protected override MdbPeopleModelDataBinder GetModelDataBinder(object context)
{
DocumentProcessingContext documentProcessingContext = context as DocumentProcessingContext;
if (documentProcessingContext != null)
{
return(MdbPeopleModelDataBinderFactory.Current.CreateInstance(documentProcessingContext.Session));
}
throw new NullDocumentProcessingContextException();
}
private void FilterMaskedContacts(DocumentProcessingContext processingContext, IDictionary <string, IInferenceRecipient> contactList)
{
if (processingContext != null && processingContext.Session != null)
{
MdbMaskedPeopleModelDataBinder mdbMaskedPeopleModelDataBinder = MdbMaskedPeopleModelDataBinderFactory.Current.CreateInstance(processingContext.Session);
MaskedPeopleModelItem modelData = mdbMaskedPeopleModelDataBinder.GetModelData();
if (modelData == null || modelData.ContactList.Count <= 0)
{
this.DiagnosticsSession.TraceDebug("Deleted items model is null or empty. Skipping removal step.", new object[0]);
return;
}
IDictionary <string, MaskedRecipient> dictionary = modelData.CreateDictionary();
List <MaskedRecipient> list = new List <MaskedRecipient>(10);
foreach (KeyValuePair <string, MaskedRecipient> keyValuePair in dictionary)
{
if (contactList.ContainsKey(keyValuePair.Key))
{
if (contactList[keyValuePair.Key].LastSentTime > keyValuePair.Value.LastMaskedFromAutoCompleteTimeUtc)
{
list.Add(keyValuePair.Value);
}
else
{
contactList.Remove(keyValuePair.Key);
}
}
}
if (list.Count > 0)
{
foreach (MaskedRecipient item in list)
{
modelData.ContactList.Remove(item);
}
this.DiagnosticsSession.TraceDebug <int>("Removed {0} out-of-date contacts from the masked model item", list.Count);
MdbModelUtils.WriteModelItem <MaskedPeopleModelItem, MdbMaskedPeopleModelDataBinder>(mdbMaskedPeopleModelDataBinder, modelData);
return;
}
}
else
{
this.DiagnosticsSession.TraceDebug("Context or Session is null. Skipping removal step.", new object[0]);
}
}
private void DetermineCaptureFlag(IInferenceRecipient recipient, DocumentProcessingContext processingContext)
{
int relevanceCategory = recipient.RelevanceCategory;
int captureFlag = recipient.CaptureFlag;
int num = 0;
if (relevanceCategory == 2147483647)
{
if (recipient.RelevanceCategoryAtLastCapture != 2147483647)
{
num = 4;
}
}
else if (recipient.RelevanceCategoryAtLastCapture == 2147483647)
{
num = 1;
}
else if (relevanceCategory != recipient.RelevanceCategoryAtLastCapture)
{
num = 2;
}
recipient.CaptureFlag = num;
if (captureFlag != num)
{
this.DiagnosticsSession.LogDiagnosticsInfo(DiagnosticsLoggingTag.Informational, string.Format("U={0} - Recipient cache entry {1} capture flag change: CF={2} CFB={3} R={4} W={5} RCC={6}", new object[]
{
(processingContext != null) ? processingContext.Session.MailboxOwner.MailboxInfo.PrimarySmtpAddress.ToString() : string.Empty,
recipient.SmtpAddress,
num,
captureFlag,
recipient.RecipientRank,
recipient.RawRecipientWeight,
recipient.RelevanceCategoryAtLastCapture
}), new object[0]);
}
}
protected override void InternalProcessDocument(DocumentContext data)
{
object obj = null;
if (data.Document.TryGetProperty(PeopleRelevanceSchema.IsBasedOnRecipientInfoData, out obj) && (bool)obj)
{
this.DiagnosticsSession.TraceDebug("RecipientInfo-based relevance is enabled. Skip processing SentItems", new object[0]);
return;
}
DocumentProcessingContext documentProcessingContext = data.AsyncResult.AsyncState as DocumentProcessingContext;
Util.ThrowOnConditionFailed(documentProcessingContext != null, "Context object passed in async call is not of type DocumentProcessingContext");
Util.ThrowOnMismatchType <Document>(data.Document, "data.Document");
Document document = data.Document as Document;
this.DiagnosticsSession.TraceDebug <IIdentity>("Processing document - {0}", document.Identity);
IList <IDocument> list = null;
IDocument property = document.GetProperty <IDocument>(PeopleRelevanceSchema.SentItemsTrainingSubDocument);
this.DiagnosticsSession.TraceDebug <int>("Count of nested documents in the training set - {0}", property.NestedDocuments.Count);
if (document.TryGetProperty(PeopleRelevanceSchema.CurrentTimeWindowNumber, out obj))
{
document.SetProperty(PeopleRelevanceSchema.TimeWindowNumberAtLastRun, (long)obj);
}
bool flag = false;
foreach (IDocument document2 in property.NestedDocuments)
{
if (base.PipelineContext != null && InferencePipelineUtil.IsAbortOnProcessingRequested((PipelineContext)base.PipelineContext))
{
flag = true;
break;
}
this.DiagnosticsSession.TraceDebug <IIdentity>("Processing mini document - {0}", document2.Identity);
using (MdbDocument mdbDocument = MdbInferenceFactory.Current.CreateFullDocument(document2, documentProcessingContext, MdbPeoplePropertyMap.Instance, PeopleRelevanceDocumentFactory.SentItemsFullDocumentProperties))
{
this.DiagnosticsSession.TraceDebug <IIdentity>("Creating full document succeeded - {0}", mdbDocument.Identity);
if (document.TryGetProperty(PeopleRelevanceSchema.ContactList, out obj))
{
mdbDocument.SetProperty(PeopleRelevanceSchema.ContactList, obj);
}
if (document.TryGetProperty(PeopleRelevanceSchema.CurrentTimeWindowStartTime, out obj))
{
mdbDocument.SetProperty(PeopleRelevanceSchema.CurrentTimeWindowStartTime, obj);
mdbDocument.SetProperty(PeopleRelevanceSchema.CurrentTimeWindowNumber, document.GetProperty <long>(PeopleRelevanceSchema.CurrentTimeWindowNumber));
}
if (document.TryGetProperty(PeopleRelevanceSchema.MailboxOwner, out obj))
{
mdbDocument.SetProperty(PeopleRelevanceSchema.MailboxOwner, obj);
}
this.DiagnosticsSession.TraceDebug <IIdentity>("Pushing full document to the nested pipeline - {0}", mdbDocument.Identity);
try
{
this.NestedPipeline.ProcessDocument(mdbDocument, documentProcessingContext);
this.DiagnosticsSession.TraceDebug <IIdentity>("Processing full document in the nested pipeline completed without exceptions - {0}", mdbDocument.Identity);
if (mdbDocument.TryGetProperty(PeopleRelevanceSchema.SentTime, out obj))
{
document2.SetProperty(PeopleRelevanceSchema.SentTime, obj);
}
else
{
this.DiagnosticsSession.TraceDebug <IIdentity>("Missing sent time property in full document after completing nested pipeline - {0}", mdbDocument.Identity);
}
IDictionary <string, IInferenceRecipient> property2 = mdbDocument.GetProperty <IDictionary <string, IInferenceRecipient> >(PeopleRelevanceSchema.ContactList);
document.SetProperty(PeopleRelevanceSchema.ContactList, property2);
ExDateTime property3 = mdbDocument.GetProperty <ExDateTime>(PeopleRelevanceSchema.CurrentTimeWindowStartTime);
document.SetProperty(PeopleRelevanceSchema.CurrentTimeWindowStartTime, property3);
long property4 = mdbDocument.GetProperty <long>(PeopleRelevanceSchema.CurrentTimeWindowNumber);
document.SetProperty(PeopleRelevanceSchema.CurrentTimeWindowNumber, property4);
}
catch (ComponentException ex)
{
if (list == null)
{
list = new List <IDocument>(property.NestedDocuments.Count);
}
this.DiagnosticsSession.TraceError <IIdentity, string>("Received an operation failed exception while processing mini document - {0} message- {1}", document2.Identity, ex.Message);
this.DiagnosticsSession.TraceDebug <IIdentity>("Adding the mini document - {0} to the failed Documents collection", document2.Identity);
this.DiagnosticsSession.LogDiagnosticsInfo(DiagnosticsLoggingTag.Warnings, string.Format("U={0} - Received an operation failed exception while processing mini document - {1} message- {2}", documentProcessingContext.Session.MailboxOwner.MailboxInfo.PrimarySmtpAddress, document2.Identity, ex.Message), new object[0]);
list.Add(document2);
}
}
}
if (flag)
{
throw new AbortOnProcessingRequestedException();
}
int num = 0;
if (list != null && list.Count > 0)
{
num = list.Count;
this.DiagnosticsSession.TraceDebug <int>("Count of failed Documents - {0}", num);
property.RemoveDocuments(list);
if (property.NestedDocuments.Count == 0)
{
throw new NestedDocumentCountZeroException();
}
}
this.DiagnosticsSession.LogDiagnosticsInfo(DiagnosticsLoggingTag.Informational, string.Format("U={0} - NestedSentItemsPipeline: {1} messages processed successfully, {2} messages failed", documentProcessingContext.Session.MailboxOwner.MailboxInfo.PrimarySmtpAddress, property.NestedDocuments.Count, num), new object[0]);
}
private void ProcessClassificationDocumentForComparisonModel(MdbDocument comparisonDocument, InferencePropertyBag comparisonDiagnostics, int comparisonVersion, MdbDocument classificationDocument, InferencePropertyBag classificationDiagnostics, Dictionary <OrderedFeatureSet, FeatureValues> cachedFeatureValues, DocumentProcessingContext processingContext, MdbInferenceRecipient ownerAsInferenceRecipient, ModelVersionBreadCrumb breadCrumb, VariantConfigurationSnapshot flightFeatures, object uniqueBodyWords, bool?conversationLoadRequired)
{
try
{
object obj;
if (classificationDiagnostics.TryGetValue(InferenceSchema.MessageClassificationTime, out obj))
{
comparisonDiagnostics.Add(InferenceSchema.MessageClassificationTime, obj);
}
if (classificationDiagnostics.TryGetValue(DocumentSchema.MailboxId, out obj))
{
comparisonDiagnostics.Add(DocumentSchema.MailboxId, obj);
}
if (classificationDiagnostics.TryGetValue(InferenceSchema.InternetMessageId, out obj))
{
comparisonDiagnostics.Add(InferenceSchema.InternetMessageId, obj);
}
if (classificationDiagnostics.TryGetValue(InferenceSchema.MessageIdentifier, out obj))
{
comparisonDiagnostics.Add(InferenceSchema.MessageIdentifier, obj);
}
if (classificationDocument.TryGetProperty(InferenceSchema.IsNewConversation, out obj))
{
comparisonDocument.SetProperty(InferenceSchema.IsNewConversation, obj);
}
if (classificationDocument.TryGetProperty(InferenceSchema.ConversationClutterInformation, out obj))
{
comparisonDocument.SetProperty(InferenceSchema.ConversationClutterInformation, obj);
}
if (classificationDocument.TryGetProperty(InferenceSchema.ConversationImportanceProperties, out obj))
{
comparisonDocument.SetProperty(InferenceSchema.ConversationImportanceProperties, obj);
}
comparisonDocument.SetProperty(InferenceSchema.SkipConversationExtraction, true);
bool flag = false;
OrderedFeatureSet featureSet = ModelConfiguration.GetModelVersionConfiguration(comparisonVersion).FeatureSet;
if (cachedFeatureValues.ContainsKey(featureSet))
{
comparisonDocument.SetProperty(InferenceSchema.ImportanceFeatureValues, cachedFeatureValues[featureSet]);
comparisonDocument.SetProperty(InferenceSchema.SkipFeatureVectorCalculation, true);
flag = true;
}
comparisonDocument.SetProperty(InferenceSchema.SkipFeatureVectorPersistance, true);
comparisonDocument.SetProperty(InferenceSchema.SkipPredictedActionsPersistance, true);
this.ProcessClassificationDocument(comparisonDocument, comparisonDiagnostics, new ModelVersionSelector.ModelVersionInfo(comparisonVersion, false), processingContext, ownerAsInferenceRecipient, breadCrumb, flightFeatures, uniqueBodyWords, conversationLoadRequired);
if (!flag && comparisonDocument.TryGetProperty(InferenceSchema.ImportanceFeatureValues, out obj))
{
cachedFeatureValues[featureSet] = (FeatureValues)obj;
}
}
catch (Exception ex)
{
if (ex is OutOfMemoryException || ex is StackOverflowException || ex is ThreadAbortException)
{
throw;
}
comparisonDiagnostics.Add(InferenceSchema.ClassificationAgentException, ex);
}
}
private void ProcessClassificationDocument(MdbDocument document, InferencePropertyBag classificationDiagnostics, ModelVersionSelector.ModelVersionInfo versionInfo, DocumentProcessingContext processingContext, MdbInferenceRecipient ownerAsInferenceRecipient, ModelVersionBreadCrumb breadCrumb, VariantConfigurationSnapshot flightFeatures, object uniqueBodyWords, bool?conversationLoadRequired)
{
document.SetProperty(InferenceSchema.MailboxOwner, ownerAsInferenceRecipient);
document.SetProperty(InferenceSchema.ClassificationDiagnostics, classificationDiagnostics);
document.SetProperty(InferenceSchema.ModelVersionToLoad, versionInfo);
document.SetProperty(InferenceSchema.ModelVersionBreadCrumb, breadCrumb);
if (conversationLoadRequired != null)
{
document.SetProperty(InferenceSchema.ConversationLoadRequired, conversationLoadRequired);
}
if (flightFeatures != null)
{
document.SetProperty(InferenceSchema.UserFlightFeatures, flightFeatures);
}
if (uniqueBodyWords != null)
{
document.SetProperty(InferenceSchema.UniqueBodyWords, uniqueBodyWords);
}
this.pipeline.ProcessDocument(document, processingContext);
}
internal bool RunClassificationPipeline(StoreDriverDeliveryEventArgsImpl argsImpl, InferencePropertyBag classificationDiagnostics, VariantConfigurationSnapshot flightFeatures, out MdbDocument document, out bool isClutter, out ConversationClutterInformation conversationClutterInformation)
{
document = null;
isClutter = false;
conversationClutterInformation = null;
InferenceClassificationAgent.tracer.TraceDebug((long)this.GetHashCode(), "Processing incoming message");
ModelVersionBreadCrumb breadCrumb = ClutterUtilities.GetModelVersionBreadCrumb(argsImpl.MailboxSession);
ModelVersionSelector versionSelector = new ModelVersionSelector(this.modelConfiguration, breadCrumb, delegate(string str)
{
this.diagnosticsSession.TraceDebug(str, new object[0]);
});
if (versionSelector.ClassificationModelVersion.Version == -2147483648)
{
classificationDiagnostics.Add(InferenceSchema.ClassificationStatus, InferenceClassificationAgentLogger.Status.Skipped);
classificationDiagnostics.Add(InferenceSchema.ClassificationStatusMessage, "ClassificationModelNotFound");
return(false);
}
MdbInferenceRecipient ownerAsInferenceRecipient;
if (string.Equals(argsImpl.MailboxSession.MailboxOwner.MailboxInfo.DisplayName, argsImpl.MailboxSession.MailboxOwner.LegacyDn, StringComparison.OrdinalIgnoreCase))
{
ownerAsInferenceRecipient = new MdbInferenceRecipient(argsImpl.MailboxSession.MailboxOwner, argsImpl.MailboxSession.Culture, argsImpl.MailboxOwner.DisplayName);
}
else
{
ownerAsInferenceRecipient = new MdbInferenceRecipient(argsImpl.MailboxSession.MailboxOwner, argsImpl.MailboxSession.Culture, null);
}
bool?conversationLoadRequired = null;
if (argsImpl.SharedPropertiesBetweenAgents != null && argsImpl.SharedPropertiesBetweenAgents.ContainsKey(ItemSchema.ConversationLoadRequiredByInference))
{
conversationLoadRequired = new bool?((bool)argsImpl.SharedPropertiesBetweenAgents[ItemSchema.ConversationLoadRequiredByInference]);
}
DocumentProcessingContext processingContext = new DocumentProcessingContext(argsImpl.MailboxSession);
MdbCompositeItemIdentity mdbCompositeItemIdentity = new MdbCompositeItemIdentity(argsImpl.MailboxSession.MdbGuid, argsImpl.MailboxSession.MailboxGuid, StoreObjectId.DummyId, 1);
MdbInMemoryDocumentAdapter mdbInMemoryDocumentAdapter = new MdbInMemoryDocumentAdapter(mdbCompositeItemIdentity, argsImpl.ReplayItem, MdbInferencePropertyMap.Instance);
document = new MdbDocument(mdbCompositeItemIdentity, DocumentOperation.Insert, mdbInMemoryDocumentAdapter);
object uniqueBodyWords;
document.TryGetProperty(InferenceSchema.UniqueBodyWordsFromProvider, out uniqueBodyWords);
this.ProcessClassificationDocument(document, classificationDiagnostics, versionSelector.ClassificationModelVersion, processingContext, ownerAsInferenceRecipient, breadCrumb, flightFeatures, uniqueBodyWords, conversationLoadRequired);
IItem inMemoryItem = mdbInMemoryDocumentAdapter.InMemoryItem;
if (inMemoryItem == null)
{
InferenceClassificationAgent.tracer.TraceError((long)this.GetHashCode(), "The InMemoryItem of the inMemDocAdapter was null");
classificationDiagnostics.Add(InferenceSchema.ClassificationStatus, InferenceClassificationAgentLogger.Status.Skipped);
classificationDiagnostics.Add(InferenceSchema.ClassificationStatusMessage, "InMemoryItemNull");
return(false);
}
object obj;
if (document.TryGetProperty(InferenceSchema.ConversationClutterInformation, out obj))
{
conversationClutterInformation = (obj as ConversationClutterInformation);
}
classificationDiagnostics.Add(InferenceSchema.ConversationClutterInformation, conversationClutterInformation);
classificationDiagnostics.Add(InferenceSchema.MarkedAsBulk, InferenceClassificationAgent.IsBulkMail(argsImpl, argsImpl.MailboxOwner));
isClutter = document.GetProperty <bool>(InferenceSchema.ComputedClutterValue);
foreach (StorePropertyDefinition propertyDefinition in InferenceClassificationAgent.PropertiesToReplicate)
{
InferenceClassificationAgent.CopyIfValid(propertyDefinition, inMemoryItem, argsImpl.PropertiesForAllMessageCopies);
}
if (this.classificationComparisonLogger != null && flightFeatures != null && flightFeatures.Inference.InferenceModelComparison.Enabled)
{
IEnumerable <int> enumerable = from entry in versionSelector.TrainingModelVersions
where entry.Version != versionSelector.ClassificationModelVersion.Version && (breadCrumb.Contains((short)entry.Version, ModelVersionBreadCrumb.VersionType.Ready) || breadCrumb.Contains((short)entry.Version, ModelVersionBreadCrumb.VersionType.NotReady))
select entry.Version;
if (enumerable != null && enumerable.Any <int>())
{
Dictionary <OrderedFeatureSet, FeatureValues> dictionary = new Dictionary <OrderedFeatureSet, FeatureValues>();
dictionary.Add(ModelConfiguration.GetModelVersionConfiguration(versionSelector.ClassificationModelVersion.Version).FeatureSet, document.GetProperty <FeatureValues>(InferenceSchema.ImportanceFeatureValues));
this.classificationComparisonLogger.LogModelComparisonData(document, classificationDiagnostics);
foreach (int comparisonVersion in enumerable)
{
MdbDocument mdbDocument = new MdbDocument(mdbCompositeItemIdentity, DocumentOperation.Insert, mdbInMemoryDocumentAdapter);
InferencePropertyBag inferencePropertyBag = new InferencePropertyBag();
this.ProcessClassificationDocumentForComparisonModel(mdbDocument, inferencePropertyBag, comparisonVersion, document, classificationDiagnostics, dictionary, processingContext, ownerAsInferenceRecipient, breadCrumb, flightFeatures, uniqueBodyWords, conversationLoadRequired);
this.classificationComparisonLogger.LogModelComparisonData(mdbDocument, inferencePropertyBag);
}
}
}
return(true);
}
protected override void InternalProcessDocument(DocumentContext data)
{
this.DiagnosticsSession.TraceDebug <IIdentity>("Processing document - {0}", data.Document.Identity);
DocumentProcessingContext processingContext = (DocumentProcessingContext)data.AsyncResult.AsyncState;
IDocument document = data.Document;
object obj;
if (document.TryGetProperty(PeopleRelevanceSchema.IsBasedOnRecipientInfoData, out obj) && (bool)obj)
{
object obj2;
if (!document.TryGetProperty(PeopleRelevanceSchema.RecipientInfoEnumerable, out obj2))
{
this.DiagnosticsSession.TraceDebug("RecipientInfo list doesn't exist. Nothing to process", new object[0]);
return;
}
IEnumerable <IRecipientInfo> enumerable = obj2 as IEnumerable <IRecipientInfo>;
if (enumerable != null && !enumerable.Any <IRecipientInfo>())
{
this.DiagnosticsSession.TraceDebug("RecipientInfo list is empty. Nothing to process", new object[0]);
return;
}
}
else
{
IDocument property = document.GetProperty <IDocument>(PeopleRelevanceSchema.SentItemsTrainingSubDocument);
Util.ThrowOnConditionFailed(property.NestedDocuments != null, "Invalid training document. NestedDocuments is null.");
if (property.NestedDocuments.Count == 0)
{
this.DiagnosticsSession.TraceDebug("NestedDocuments is empty. Nothing to process", new object[0]);
return;
}
}
IDictionary <string, IInferenceRecipient> dictionary = null;
if (!data.Document.TryGetProperty(PeopleRelevanceSchema.ContactList, out obj))
{
this.DiagnosticsSession.TraceDebug("Contact list is not present. Nothing to process", new object[0]);
return;
}
dictionary = (obj as IDictionary <string, IInferenceRecipient>);
if (dictionary.Count == 0)
{
this.DiagnosticsSession.TraceDebug("Contact list is empty. Nothing to process", new object[0]);
return;
}
this.FilterMaskedContacts(processingContext, dictionary);
long property2 = data.Document.GetProperty <long>(PeopleRelevanceSchema.CurrentTimeWindowNumber);
long timeWindowNumberAtLastRun = 0L;
if (data.Document.TryGetProperty(PeopleRelevanceSchema.TimeWindowNumberAtLastRun, out obj))
{
timeWindowNumberAtLastRun = (long)obj;
}
List <string> list = new List <string>();
foreach (IInferenceRecipient inferenceRecipient in dictionary.Values)
{
PeopleRelevanceClassifier.ApplyDecay(inferenceRecipient, property2, timeWindowNumberAtLastRun);
if (inferenceRecipient.RawRecipientWeight > 0.0)
{
if (!string.IsNullOrEmpty(inferenceRecipient.SmtpAddress))
{
list.Add(inferenceRecipient.SmtpAddress.ToLower(CultureInfo.InvariantCulture));
}
else
{
this.DiagnosticsSession.TraceDebug <string>("Found Contact without SMTP: {0}", inferenceRecipient.Identity.ToString());
}
}
}
PeopleRelevanceClassifier.RankRecipients(dictionary, list);
foreach (IInferenceRecipient recipient in dictionary.Values)
{
this.DetermineCaptureFlag(recipient, processingContext);
}
this.DiagnosticsSession.TraceDebug("Store updated recipient properties in the document", new object[0]);
data.Document.SetProperty(PeopleRelevanceSchema.ContactList, dictionary);
data.Document.SetProperty(PeopleRelevanceSchema.TopRankedContacts, list);
}
