private void ProcessClassificationDocument(MdbDocument document, InferencePropertyBag classificationDiagnostics, ModelVersionSelector.ModelVersionInfo versionInfo, DocumentProcessingContext processingContext, MdbInferenceRecipient ownerAsInferenceRecipient, ModelVersionBreadCrumb breadCrumb, VariantConfigurationSnapshot flightFeatures, object uniqueBodyWords, bool?conversationLoadRequired) { document.SetProperty(InferenceSchema.MailboxOwner, ownerAsInferenceRecipient); document.SetProperty(InferenceSchema.ClassificationDiagnostics, classificationDiagnostics); document.SetProperty(InferenceSchema.ModelVersionToLoad, versionInfo); document.SetProperty(InferenceSchema.ModelVersionBreadCrumb, breadCrumb); if (conversationLoadRequired != null) { document.SetProperty(InferenceSchema.ConversationLoadRequired, conversationLoadRequired); } if (flightFeatures != null) { document.SetProperty(InferenceSchema.UserFlightFeatures, flightFeatures); } if (uniqueBodyWords != null) { document.SetProperty(InferenceSchema.UniqueBodyWords, uniqueBodyWords); } this.pipeline.ProcessDocument(document, processingContext); }
private void ProcessClassificationDocumentForComparisonModel(MdbDocument comparisonDocument, InferencePropertyBag comparisonDiagnostics, int comparisonVersion, MdbDocument classificationDocument, InferencePropertyBag classificationDiagnostics, Dictionary <OrderedFeatureSet, FeatureValues> cachedFeatureValues, DocumentProcessingContext processingContext, MdbInferenceRecipient ownerAsInferenceRecipient, ModelVersionBreadCrumb breadCrumb, VariantConfigurationSnapshot flightFeatures, object uniqueBodyWords, bool?conversationLoadRequired) { try { object obj; if (classificationDiagnostics.TryGetValue(InferenceSchema.MessageClassificationTime, out obj)) { comparisonDiagnostics.Add(InferenceSchema.MessageClassificationTime, obj); } if (classificationDiagnostics.TryGetValue(DocumentSchema.MailboxId, out obj)) { comparisonDiagnostics.Add(DocumentSchema.MailboxId, obj); } if (classificationDiagnostics.TryGetValue(InferenceSchema.InternetMessageId, out obj)) { comparisonDiagnostics.Add(InferenceSchema.InternetMessageId, obj); } if (classificationDiagnostics.TryGetValue(InferenceSchema.MessageIdentifier, out obj)) { comparisonDiagnostics.Add(InferenceSchema.MessageIdentifier, obj); } if (classificationDocument.TryGetProperty(InferenceSchema.IsNewConversation, out obj)) { comparisonDocument.SetProperty(InferenceSchema.IsNewConversation, obj); } if (classificationDocument.TryGetProperty(InferenceSchema.ConversationClutterInformation, out obj)) { comparisonDocument.SetProperty(InferenceSchema.ConversationClutterInformation, obj); } if (classificationDocument.TryGetProperty(InferenceSchema.ConversationImportanceProperties, out obj)) { comparisonDocument.SetProperty(InferenceSchema.ConversationImportanceProperties, obj); } comparisonDocument.SetProperty(InferenceSchema.SkipConversationExtraction, true); bool flag = false; OrderedFeatureSet featureSet = ModelConfiguration.GetModelVersionConfiguration(comparisonVersion).FeatureSet; if (cachedFeatureValues.ContainsKey(featureSet)) { comparisonDocument.SetProperty(InferenceSchema.ImportanceFeatureValues, cachedFeatureValues[featureSet]); comparisonDocument.SetProperty(InferenceSchema.SkipFeatureVectorCalculation, true); flag = true; } comparisonDocument.SetProperty(InferenceSchema.SkipFeatureVectorPersistance, true); comparisonDocument.SetProperty(InferenceSchema.SkipPredictedActionsPersistance, true); this.ProcessClassificationDocument(comparisonDocument, comparisonDiagnostics, new ModelVersionSelector.ModelVersionInfo(comparisonVersion, false), processingContext, ownerAsInferenceRecipient, breadCrumb, flightFeatures, uniqueBodyWords, conversationLoadRequired); if (!flag && comparisonDocument.TryGetProperty(InferenceSchema.ImportanceFeatureValues, out obj)) { cachedFeatureValues[featureSet] = (FeatureValues)obj; } } catch (Exception ex) { if (ex is OutOfMemoryException || ex is StackOverflowException || ex is ThreadAbortException) { throw; } comparisonDiagnostics.Add(InferenceSchema.ClassificationAgentException, ex); } }
internal bool RunClassificationPipeline(StoreDriverDeliveryEventArgsImpl argsImpl, InferencePropertyBag classificationDiagnostics, VariantConfigurationSnapshot flightFeatures, out MdbDocument document, out bool isClutter, out ConversationClutterInformation conversationClutterInformation) { document = null; isClutter = false; conversationClutterInformation = null; InferenceClassificationAgent.tracer.TraceDebug((long)this.GetHashCode(), "Processing incoming message"); ModelVersionBreadCrumb breadCrumb = ClutterUtilities.GetModelVersionBreadCrumb(argsImpl.MailboxSession); ModelVersionSelector versionSelector = new ModelVersionSelector(this.modelConfiguration, breadCrumb, delegate(string str) { this.diagnosticsSession.TraceDebug(str, new object[0]); }); if (versionSelector.ClassificationModelVersion.Version == -2147483648) { classificationDiagnostics.Add(InferenceSchema.ClassificationStatus, InferenceClassificationAgentLogger.Status.Skipped); classificationDiagnostics.Add(InferenceSchema.ClassificationStatusMessage, "ClassificationModelNotFound"); return(false); } MdbInferenceRecipient ownerAsInferenceRecipient; if (string.Equals(argsImpl.MailboxSession.MailboxOwner.MailboxInfo.DisplayName, argsImpl.MailboxSession.MailboxOwner.LegacyDn, StringComparison.OrdinalIgnoreCase)) { ownerAsInferenceRecipient = new MdbInferenceRecipient(argsImpl.MailboxSession.MailboxOwner, argsImpl.MailboxSession.Culture, argsImpl.MailboxOwner.DisplayName); } else { ownerAsInferenceRecipient = new MdbInferenceRecipient(argsImpl.MailboxSession.MailboxOwner, argsImpl.MailboxSession.Culture, null); } bool?conversationLoadRequired = null; if (argsImpl.SharedPropertiesBetweenAgents != null && argsImpl.SharedPropertiesBetweenAgents.ContainsKey(ItemSchema.ConversationLoadRequiredByInference)) { conversationLoadRequired = new bool?((bool)argsImpl.SharedPropertiesBetweenAgents[ItemSchema.ConversationLoadRequiredByInference]); } DocumentProcessingContext processingContext = new DocumentProcessingContext(argsImpl.MailboxSession); MdbCompositeItemIdentity mdbCompositeItemIdentity = new MdbCompositeItemIdentity(argsImpl.MailboxSession.MdbGuid, argsImpl.MailboxSession.MailboxGuid, StoreObjectId.DummyId, 1); MdbInMemoryDocumentAdapter mdbInMemoryDocumentAdapter = new MdbInMemoryDocumentAdapter(mdbCompositeItemIdentity, argsImpl.ReplayItem, MdbInferencePropertyMap.Instance); document = new MdbDocument(mdbCompositeItemIdentity, DocumentOperation.Insert, mdbInMemoryDocumentAdapter); object uniqueBodyWords; document.TryGetProperty(InferenceSchema.UniqueBodyWordsFromProvider, out uniqueBodyWords); this.ProcessClassificationDocument(document, classificationDiagnostics, versionSelector.ClassificationModelVersion, processingContext, ownerAsInferenceRecipient, breadCrumb, flightFeatures, uniqueBodyWords, conversationLoadRequired); IItem inMemoryItem = mdbInMemoryDocumentAdapter.InMemoryItem; if (inMemoryItem == null) { InferenceClassificationAgent.tracer.TraceError((long)this.GetHashCode(), "The InMemoryItem of the inMemDocAdapter was null"); classificationDiagnostics.Add(InferenceSchema.ClassificationStatus, InferenceClassificationAgentLogger.Status.Skipped); classificationDiagnostics.Add(InferenceSchema.ClassificationStatusMessage, "InMemoryItemNull"); return(false); } object obj; if (document.TryGetProperty(InferenceSchema.ConversationClutterInformation, out obj)) { conversationClutterInformation = (obj as ConversationClutterInformation); } classificationDiagnostics.Add(InferenceSchema.ConversationClutterInformation, conversationClutterInformation); classificationDiagnostics.Add(InferenceSchema.MarkedAsBulk, InferenceClassificationAgent.IsBulkMail(argsImpl, argsImpl.MailboxOwner)); isClutter = document.GetProperty <bool>(InferenceSchema.ComputedClutterValue); foreach (StorePropertyDefinition propertyDefinition in InferenceClassificationAgent.PropertiesToReplicate) { InferenceClassificationAgent.CopyIfValid(propertyDefinition, inMemoryItem, argsImpl.PropertiesForAllMessageCopies); } if (this.classificationComparisonLogger != null && flightFeatures != null && flightFeatures.Inference.InferenceModelComparison.Enabled) { IEnumerable <int> enumerable = from entry in versionSelector.TrainingModelVersions where entry.Version != versionSelector.ClassificationModelVersion.Version && (breadCrumb.Contains((short)entry.Version, ModelVersionBreadCrumb.VersionType.Ready) || breadCrumb.Contains((short)entry.Version, ModelVersionBreadCrumb.VersionType.NotReady)) select entry.Version; if (enumerable != null && enumerable.Any <int>()) { Dictionary <OrderedFeatureSet, FeatureValues> dictionary = new Dictionary <OrderedFeatureSet, FeatureValues>(); dictionary.Add(ModelConfiguration.GetModelVersionConfiguration(versionSelector.ClassificationModelVersion.Version).FeatureSet, document.GetProperty <FeatureValues>(InferenceSchema.ImportanceFeatureValues)); this.classificationComparisonLogger.LogModelComparisonData(document, classificationDiagnostics); foreach (int comparisonVersion in enumerable) { MdbDocument mdbDocument = new MdbDocument(mdbCompositeItemIdentity, DocumentOperation.Insert, mdbInMemoryDocumentAdapter); InferencePropertyBag inferencePropertyBag = new InferencePropertyBag(); this.ProcessClassificationDocumentForComparisonModel(mdbDocument, inferencePropertyBag, comparisonVersion, document, classificationDiagnostics, dictionary, processingContext, ownerAsInferenceRecipient, breadCrumb, flightFeatures, uniqueBodyWords, conversationLoadRequired); this.classificationComparisonLogger.LogModelComparisonData(mdbDocument, inferencePropertyBag); } } } return(true); }