private void ProcessClassificationDocumentForComparisonModel(MdbDocument comparisonDocument, InferencePropertyBag comparisonDiagnostics, int comparisonVersion, MdbDocument classificationDocument, InferencePropertyBag classificationDiagnostics, Dictionary <OrderedFeatureSet, FeatureValues> cachedFeatureValues, DocumentProcessingContext processingContext, MdbInferenceRecipient ownerAsInferenceRecipient, ModelVersionBreadCrumb breadCrumb, VariantConfigurationSnapshot flightFeatures, object uniqueBodyWords, bool?conversationLoadRequired) { try { object obj; if (classificationDiagnostics.TryGetValue(InferenceSchema.MessageClassificationTime, out obj)) { comparisonDiagnostics.Add(InferenceSchema.MessageClassificationTime, obj); } if (classificationDiagnostics.TryGetValue(DocumentSchema.MailboxId, out obj)) { comparisonDiagnostics.Add(DocumentSchema.MailboxId, obj); } if (classificationDiagnostics.TryGetValue(InferenceSchema.InternetMessageId, out obj)) { comparisonDiagnostics.Add(InferenceSchema.InternetMessageId, obj); } if (classificationDiagnostics.TryGetValue(InferenceSchema.MessageIdentifier, out obj)) { comparisonDiagnostics.Add(InferenceSchema.MessageIdentifier, obj); } if (classificationDocument.TryGetProperty(InferenceSchema.IsNewConversation, out obj)) { comparisonDocument.SetProperty(InferenceSchema.IsNewConversation, obj); } if (classificationDocument.TryGetProperty(InferenceSchema.ConversationClutterInformation, out obj)) { comparisonDocument.SetProperty(InferenceSchema.ConversationClutterInformation, obj); } if (classificationDocument.TryGetProperty(InferenceSchema.ConversationImportanceProperties, out obj)) { comparisonDocument.SetProperty(InferenceSchema.ConversationImportanceProperties, obj); } comparisonDocument.SetProperty(InferenceSchema.SkipConversationExtraction, true); bool flag = false; OrderedFeatureSet featureSet = ModelConfiguration.GetModelVersionConfiguration(comparisonVersion).FeatureSet; if (cachedFeatureValues.ContainsKey(featureSet)) { comparisonDocument.SetProperty(InferenceSchema.ImportanceFeatureValues, cachedFeatureValues[featureSet]); comparisonDocument.SetProperty(InferenceSchema.SkipFeatureVectorCalculation, true); flag = true; } comparisonDocument.SetProperty(InferenceSchema.SkipFeatureVectorPersistance, true); comparisonDocument.SetProperty(InferenceSchema.SkipPredictedActionsPersistance, true); this.ProcessClassificationDocument(comparisonDocument, comparisonDiagnostics, new ModelVersionSelector.ModelVersionInfo(comparisonVersion, false), processingContext, ownerAsInferenceRecipient, breadCrumb, flightFeatures, uniqueBodyWords, conversationLoadRequired); if (!flag && comparisonDocument.TryGetProperty(InferenceSchema.ImportanceFeatureValues, out obj)) { cachedFeatureValues[featureSet] = (FeatureValues)obj; } } catch (Exception ex) { if (ex is OutOfMemoryException || ex is StackOverflowException || ex is ThreadAbortException) { throw; } comparisonDiagnostics.Add(InferenceSchema.ClassificationAgentException, ex); } }
private void ProcessClassificationDocument(MdbDocument document, InferencePropertyBag classificationDiagnostics, ModelVersionSelector.ModelVersionInfo versionInfo, DocumentProcessingContext processingContext, MdbInferenceRecipient ownerAsInferenceRecipient, ModelVersionBreadCrumb breadCrumb, VariantConfigurationSnapshot flightFeatures, object uniqueBodyWords, bool?conversationLoadRequired) { document.SetProperty(InferenceSchema.MailboxOwner, ownerAsInferenceRecipient); document.SetProperty(InferenceSchema.ClassificationDiagnostics, classificationDiagnostics); document.SetProperty(InferenceSchema.ModelVersionToLoad, versionInfo); document.SetProperty(InferenceSchema.ModelVersionBreadCrumb, breadCrumb); if (conversationLoadRequired != null) { document.SetProperty(InferenceSchema.ConversationLoadRequired, conversationLoadRequired); } if (flightFeatures != null) { document.SetProperty(InferenceSchema.UserFlightFeatures, flightFeatures); } if (uniqueBodyWords != null) { document.SetProperty(InferenceSchema.UniqueBodyWords, uniqueBodyWords); } this.pipeline.ProcessDocument(document, processingContext); }
protected override void InternalProcessDocument(DocumentContext data) { object obj = null; if (data.Document.TryGetProperty(PeopleRelevanceSchema.IsBasedOnRecipientInfoData, out obj) && (bool)obj) { this.DiagnosticsSession.TraceDebug("RecipientInfo-based relevance is enabled. Skip processing SentItems", new object[0]); return; } DocumentProcessingContext documentProcessingContext = data.AsyncResult.AsyncState as DocumentProcessingContext; Util.ThrowOnConditionFailed(documentProcessingContext != null, "Context object passed in async call is not of type DocumentProcessingContext"); Util.ThrowOnMismatchType <Document>(data.Document, "data.Document"); Document document = data.Document as Document; this.DiagnosticsSession.TraceDebug <IIdentity>("Processing document - {0}", document.Identity); IList <IDocument> list = null; IDocument property = document.GetProperty <IDocument>(PeopleRelevanceSchema.SentItemsTrainingSubDocument); this.DiagnosticsSession.TraceDebug <int>("Count of nested documents in the training set - {0}", property.NestedDocuments.Count); if (document.TryGetProperty(PeopleRelevanceSchema.CurrentTimeWindowNumber, out obj)) { document.SetProperty(PeopleRelevanceSchema.TimeWindowNumberAtLastRun, (long)obj); } bool flag = false; foreach (IDocument document2 in property.NestedDocuments) { if (base.PipelineContext != null && InferencePipelineUtil.IsAbortOnProcessingRequested((PipelineContext)base.PipelineContext)) { flag = true; break; } this.DiagnosticsSession.TraceDebug <IIdentity>("Processing mini document - {0}", document2.Identity); using (MdbDocument mdbDocument = MdbInferenceFactory.Current.CreateFullDocument(document2, documentProcessingContext, MdbPeoplePropertyMap.Instance, PeopleRelevanceDocumentFactory.SentItemsFullDocumentProperties)) { this.DiagnosticsSession.TraceDebug <IIdentity>("Creating full document succeeded - {0}", mdbDocument.Identity); if (document.TryGetProperty(PeopleRelevanceSchema.ContactList, out obj)) { mdbDocument.SetProperty(PeopleRelevanceSchema.ContactList, obj); } if (document.TryGetProperty(PeopleRelevanceSchema.CurrentTimeWindowStartTime, out obj)) { mdbDocument.SetProperty(PeopleRelevanceSchema.CurrentTimeWindowStartTime, obj); mdbDocument.SetProperty(PeopleRelevanceSchema.CurrentTimeWindowNumber, document.GetProperty <long>(PeopleRelevanceSchema.CurrentTimeWindowNumber)); } if (document.TryGetProperty(PeopleRelevanceSchema.MailboxOwner, out obj)) { mdbDocument.SetProperty(PeopleRelevanceSchema.MailboxOwner, obj); } this.DiagnosticsSession.TraceDebug <IIdentity>("Pushing full document to the nested pipeline - {0}", mdbDocument.Identity); try { this.NestedPipeline.ProcessDocument(mdbDocument, documentProcessingContext); this.DiagnosticsSession.TraceDebug <IIdentity>("Processing full document in the nested pipeline completed without exceptions - {0}", mdbDocument.Identity); if (mdbDocument.TryGetProperty(PeopleRelevanceSchema.SentTime, out obj)) { document2.SetProperty(PeopleRelevanceSchema.SentTime, obj); } else { this.DiagnosticsSession.TraceDebug <IIdentity>("Missing sent time property in full document after completing nested pipeline - {0}", mdbDocument.Identity); } IDictionary <string, IInferenceRecipient> property2 = mdbDocument.GetProperty <IDictionary <string, IInferenceRecipient> >(PeopleRelevanceSchema.ContactList); document.SetProperty(PeopleRelevanceSchema.ContactList, property2); ExDateTime property3 = mdbDocument.GetProperty <ExDateTime>(PeopleRelevanceSchema.CurrentTimeWindowStartTime); document.SetProperty(PeopleRelevanceSchema.CurrentTimeWindowStartTime, property3); long property4 = mdbDocument.GetProperty <long>(PeopleRelevanceSchema.CurrentTimeWindowNumber); document.SetProperty(PeopleRelevanceSchema.CurrentTimeWindowNumber, property4); } catch (ComponentException ex) { if (list == null) { list = new List <IDocument>(property.NestedDocuments.Count); } this.DiagnosticsSession.TraceError <IIdentity, string>("Received an operation failed exception while processing mini document - {0} message- {1}", document2.Identity, ex.Message); this.DiagnosticsSession.TraceDebug <IIdentity>("Adding the mini document - {0} to the failed Documents collection", document2.Identity); this.DiagnosticsSession.LogDiagnosticsInfo(DiagnosticsLoggingTag.Warnings, string.Format("U={0} - Received an operation failed exception while processing mini document - {1} message- {2}", documentProcessingContext.Session.MailboxOwner.MailboxInfo.PrimarySmtpAddress, document2.Identity, ex.Message), new object[0]); list.Add(document2); } } } if (flag) { throw new AbortOnProcessingRequestedException(); } int num = 0; if (list != null && list.Count > 0) { num = list.Count; this.DiagnosticsSession.TraceDebug <int>("Count of failed Documents - {0}", num); property.RemoveDocuments(list); if (property.NestedDocuments.Count == 0) { throw new NestedDocumentCountZeroException(); } } this.DiagnosticsSession.LogDiagnosticsInfo(DiagnosticsLoggingTag.Informational, string.Format("U={0} - NestedSentItemsPipeline: {1} messages processed successfully, {2} messages failed", documentProcessingContext.Session.MailboxOwner.MailboxInfo.PrimarySmtpAddress, property.NestedDocuments.Count, num), new object[0]); }