public static async Task <Guid> RegisterRfmModelTrainingTaskChainAsync( this ITaskManager taskManager, ModelTrainingTaskOptions modelTrainingOptions, IDataSourceOptionsCollection dataSourceOptions, TimeSpan expiresAfter, IEnumerable <Guid> prerequisiteTaskIds) { var projectionDictionary = new ProjectionWorkerOptionsDictionary( modelTrainingOptions.ModelEntityTypeString, modelTrainingOptions.ModelTypeString, expiresAfter, modelTrainingOptions.SchemaName, modelTrainingOptions.ModelOptions); Guid guid = await taskManager.RegisterDistributedTaskAsync(dataSourceOptions, projectionDictionary, prerequisiteTaskIds, expiresAfter).ConfigureAwait(false); var mergeTasks = new List <Task <Guid> >(); foreach (var targetTableNames in modelTrainingOptions.SourceTargetTableNamesMap) { var optionsDictionary1 = new MergeWorkerOptionsDictionary(targetTableNames.Value, targetTableNames.Key, expiresAfter, modelTrainingOptions.SchemaName); ITaskManager taskManager1 = taskManager; var optionsDictionary2 = optionsDictionary1; Guid[] guidArray = { guid }; TimeSpan expiresAfter1 = expiresAfter; mergeTasks.Add(taskManager1.RegisterDeferredTaskAsync(optionsDictionary2, guidArray, expiresAfter1)); } await Task.WhenAll(mergeTasks).ConfigureAwait(false); var trainTaskId = await taskManager.RegisterDeferredTaskAsync(new RfmTrainingWorkerOptionsDictionary(modelTrainingOptions.ModelEntityTypeString, modelTrainingOptions.ModelTypeString, modelTrainingOptions.SchemaName, modelTrainingOptions.SourceTargetTableNamesMap.Values.ToList(), modelTrainingOptions.ModelOptions), mergeTasks.Select(t => t.Result), expiresAfter).ConfigureAwait(false); return(trainTaskId); }
public async Task <object> RegisterTasks() { //In a core role like Content Management, we can retrive Task Manager like below var taskManager = ServiceLocator.ServiceProvider.GetService <ITaskManager>(); using (XConnectClient client = Sitecore.XConnect.Client.Configuration.SitecoreXConnectClientConfiguration.GetClient()) { var query = client.Contacts.Where(contact => contact.Interactions.Any(interaction => interaction.Events.OfType <ProductPurchasedOutcome>().Any() && interaction.StartDateTime > DateTime.UtcNow.AddDays(-1) //Only for outcome happened in past 24 hours ) ); var expandOptions = new ContactExpandOptions { Interactions = new RelatedInteractionsExpandOptions() }; query = query.WithExpandOptions(expandOptions); var searchRequest = query.GetSearchRequest(); // Task for projection var dataSourceOptions = new ContactSearchDataSourceOptionsDictionary( searchRequest, // searchRequest 30, // maxBatchSize 50 // defaultSplitItemCount ); var projectionOptions = new ContactProjectionWorkerOptionsDictionary( typeof(ProductRecommendationModel).AssemblyQualifiedName, // modelTypeString TimeSpan.FromMinutes(10), // timeToLive "recommendation", // schemaName new Dictionary <string, string> // modelOptions { { ProductRecommendationModel.OptionTableName, "contactProducts" } } ); var projectionTaskId = await taskManager.RegisterDistributedTaskAsync( dataSourceOptions, // datasourceOptions projectionOptions, // workerOptions null, // prerequisiteTaskIds TimeSpan.FromMinutes(10) // expiresAfter ); // Task for merge var mergeOptions = new MergeWorkerOptionsDictionary( "contactProductsFinal", // tableName "contactProducts", // prefix TimeSpan.FromMinutes(10), // timeToLive "recommendation" // schemaName ); var mergeTaskId = await taskManager.RegisterDeferredTaskAsync( mergeOptions, // workerOptions new[] // prerequisiteTaskIds { projectionTaskId }, TimeSpan.FromMinutes(10) // expiresAfter ); // Task for predict var workerOptions = new DeferredWorkerOptionsDictionary( typeof(ProductRecommendationWorker).AssemblyQualifiedName, // workerType new Dictionary <string, string> // options { { ProductRecommendationWorker.OptionSourceTableName, "contactProductsFinal" }, { ProductRecommendationWorker.OptionTargetTableName, "contactRecommendations" }, { ProductRecommendationWorker.OptionSchemaName, "recommendation" }, { ProductRecommendationWorker.OptionLimit, "5" } }); var recommendationTaskId = await taskManager.RegisterDeferredTaskAsync( workerOptions, // workerOptions new[] // prerequisiteTaskIds { mergeTaskId }, TimeSpan.FromMinutes(10) // expiresAfter ); // Task for storage var storageOptions = new DeferredWorkerOptionsDictionary( typeof(RecommendationFacetStorageWorker).AssemblyQualifiedName, // workerType new Dictionary <string, string> // options { { RecommendationFacetStorageWorker.OptionTableName, "contactRecommendations" }, { RecommendationFacetStorageWorker.OptionSchemaName, "recommendation" } }); var storeFacetTaskId = await taskManager.RegisterDeferredTaskAsync( storageOptions, // workerOptions new[] // prerequisiteTaskIds { recommendationTaskId }, TimeSpan.FromMinutes(10) // expiresAfter ); return(new List <Guid>() { projectionTaskId, mergeTaskId, recommendationTaskId, storeFacetTaskId }); } }
protected override async Task RecurringExecuteAsync(CancellationToken token) { using (var scope = _serviceProvider.CreateScope()) { using (var xdbContext = scope.ServiceProvider.GetService <IXdbContext>()) { // Register the project, merge, predict, storage tasks here var query = xdbContext.Contacts.Where(contact => contact.Interactions.Any(interaction => interaction.Events.OfType <ProductPurchasedOutcome>().Any() && interaction.StartDateTime > DateTime.UtcNow.AddHours(-1) ) ); var expandOptions = new ContactExpandOptions { Interactions = new RelatedInteractionsExpandOptions() }; query = query.WithExpandOptions(expandOptions); var searchRequest = query.GetSearchRequest(); // Task for projection var dataSourceOptions = new ContactSearchDataSourceOptionsDictionary( searchRequest, // searchRequest 30, // maxBatchSize 50 // defaultSplitItemCount ); var projectionOptions = new ContactProjectionWorkerOptionsDictionary( typeof(ProductRecommendationModel).AssemblyQualifiedName, // modelTypeString TimeSpan.FromMinutes(10), // timeToLive "recommendation", // schemaName new Dictionary <string, string> // modelOptions { { ProductRecommendationModel.OptionTableName, "contactProducts" } } ); var projectionTaskId = await _taskManager.RegisterDistributedTaskAsync( dataSourceOptions, // datasourceOptions projectionOptions, // workerOptions null, // prerequisiteTaskIds TimeSpan.FromMinutes(10) // expiresAfter ); // Task for merge var mergeOptions = new MergeWorkerOptionsDictionary( "contactProductsFinal", // tableName "contactProducts", // prefix TimeSpan.FromMinutes(10), // timeToLive "recommendation" // schemaName ); var mergeTaskId = await _taskManager.RegisterDeferredTaskAsync( mergeOptions, // workerOptions new[] // prerequisiteTaskIds { projectionTaskId }, TimeSpan.FromMinutes(10) // expiresAfter ); // Task for predict var workerOptions = new DeferredWorkerOptionsDictionary( typeof(ProductRecommendationWorker).AssemblyQualifiedName, // workerType new Dictionary <string, string> // options { { ProductRecommendationWorker.OptionSourceTableName, "contactProductsFinal" }, { ProductRecommendationWorker.OptionTargetTableName, "contactRecommendations" }, { ProductRecommendationWorker.OptionSchemaName, "recommendation" }, { ProductRecommendationWorker.OptionLimit, "5" } }); var recommendationTaskId = await _taskManager.RegisterDeferredTaskAsync( workerOptions, // workerOptions new[] // prerequisiteTaskIds { mergeTaskId }, TimeSpan.FromMinutes(10) // expiresAfter ); // Task for storage var storageOptions = new DeferredWorkerOptionsDictionary( typeof(RecommendationFacetStorageWorker).AssemblyQualifiedName, // workerType new Dictionary <string, string> // options { { RecommendationFacetStorageWorker.OptionTableName, "contactRecommendations" }, { RecommendationFacetStorageWorker.OptionSchemaName, "recommendation" } }); var storeFacetTaskId = await _taskManager.RegisterDeferredTaskAsync( storageOptions, // workerOptions new[] // prerequisiteTaskIds { recommendationTaskId }, TimeSpan.FromMinutes(10) // expiresAfter ); _logger.LogInformation("***projection task ID: " + projectionTaskId); _logger.LogInformation("***merge task ID: " + mergeTaskId); _logger.LogInformation("***predict task ID: " + recommendationTaskId); _logger.LogInformation("***storage task ID: " + storeFacetTaskId); } } }
public static async Task RegisterRfmModelTaskChainAsync( this ITaskManager taskManager, TimeSpan expiresAfter) { // Define workers parameters // datasource for PurchaseOutcomeModel projection var interactionDataSourceOptionsDictionary = new InteractionDataSourceOptionsDictionary(new InteractionExpandOptions(IpInfo.DefaultFacetKey), 5, 10); // datasource for ContactModel protection var contactDataSourceOptionsDictionary = new ContactDataSourceOptionsDictionary(new ContactExpandOptions(PersonalInformation.DefaultFacetKey, EmailAddressList.DefaultFacetKey, ContactBehaviorProfile.DefaultFacetKey, RfmContactFacet.DefaultFacetKey) , 5, 10); var modelTrainingOptions = new ModelTrainingTaskOptions( // assembly name of our processing engine model (PurchaseInteractionModel:IModel<Interaction>) typeof(PurchaseInteractionModel).AssemblyQualifiedName, // assembly name of entity for our processing engine model (PurchaseInteractionModel:IModel<Interaction>) typeof(Interaction).AssemblyQualifiedName, // custom options that we pass to PurchaseInteractionModel new Dictionary <string, string> { ["TestCaseId"] = "Id" }, // projection tableName of PurchaseOutcomeModel, must be equal to first parameter of 'CreateTabular' method => PurchaseOutcomeModel.cs: CreateTabular("PurchaseOutcome", ...) "PurchaseOutcome", // name of resulted table (any name) "DemoResultTable"); var projectionDictionary = new ProjectionWorkerOptionsDictionary( modelTrainingOptions.ModelEntityTypeString, modelTrainingOptions.ModelTypeString, expiresAfter, modelTrainingOptions.SchemaName, modelTrainingOptions.ModelOptions); var evaluationDictionary = new EvaluationWorkerOptionsDictionary( typeof(RfmEvaluationWorker).AssemblyQualifiedName, typeof(ContactModel).AssemblyQualifiedName, new Dictionary <string, string> { ["TestCaseId"] = "Id" }, "Evaluator.Schema", expiresAfter); // Register chain of Tasks // 1) Register Projection-worker Guid projectionTaskId = await taskManager.RegisterDistributedTaskAsync( interactionDataSourceOptionsDictionary, projectionDictionary, // no prerequisite tasks Enumerable.Empty <Guid>(), expiresAfter).ConfigureAwait(false); // 2) Register Merge-worker var mergeTaskIds = new List <Task <Guid> >(); foreach (var targetTableNames in modelTrainingOptions.SourceTargetTableNamesMap) { var mergeWorkerOptionsDictionary = new MergeWorkerOptionsDictionary(targetTableNames.Value, targetTableNames.Key, expiresAfter, modelTrainingOptions.SchemaName); mergeTaskIds.Add( taskManager.RegisterDeferredTaskAsync( mergeWorkerOptionsDictionary, // execute after Projection task new[] { projectionTaskId }, expiresAfter)); } await Task.WhenAll(mergeTaskIds).ConfigureAwait(false); // 3) Register Train-worker var trainWorkerOptionsDictionary = new RfmTrainingWorkerOptionsDictionary( modelTrainingOptions.ModelEntityTypeString, modelTrainingOptions.ModelTypeString, modelTrainingOptions.SchemaName, modelTrainingOptions.SourceTargetTableNamesMap.Values.ToList(), modelTrainingOptions.ModelOptions); Guid trainTaskId = await taskManager.RegisterDeferredTaskAsync( trainWorkerOptionsDictionary, // execute after Merge task mergeTaskIds.Select(t => t.Result), expiresAfter).ConfigureAwait(false); // 4) Register Evaluate worker Guid evaluateTaskId = await taskManager.RegisterDistributedTaskAsync( contactDataSourceOptionsDictionary, evaluationDictionary, // execute after Train worker new[] { trainTaskId }, expiresAfter) .ConfigureAwait(false); }
private async Task RegisterRecommendationTaskAsync() { _taskIds.Clear(); var taskManager = GetTaskManager(); var xConnectClient = await CreateXConnectClient(); var taskTimeout = TimeSpan.FromMinutes(10); var storageTimeout = TimeSpan.FromMinutes(30); // Prepare data source query var query = xConnectClient.Contacts.Where(contact => contact.Interactions.Any(interaction => interaction.Events.OfType <MusicPurchaseOutcome>().Any() && interaction.EndDateTime > DateTime.UtcNow.AddMinutes(-TimeoutIntervalMinutes) ) ); var expandOptions = new ContactExpandOptions { Interactions = new RelatedInteractionsExpandOptions() }; query = query.WithExpandOptions(expandOptions); var searchRequest = query.GetSearchRequest(); // Task for projection var dataSourceOptions = new ContactSearchDataSourceOptionsDictionary( searchRequest, // searchRequest 30, // maxBatchSize 50 // defaultSplitItemCount ); var projectionOptions = new ContactProjectionWorkerOptionsDictionary( typeof(AlbumRecommendationModel).AssemblyQualifiedName, // modelTypeString storageTimeout, // timeToLive "recommendation", // schemaName new Dictionary <string, string> // modelOptions { { AlbumRecommendationModel.OptionTableName, "contactArtists" } } ); var projectionTaskId = await taskManager.RegisterDistributedTaskAsync( dataSourceOptions, // datasourceOptions projectionOptions, // workerOptions null, // prerequisiteTaskIds taskTimeout // expiresAfter ); _taskIds.Add(projectionTaskId); // Task for merge var mergeOptions = new MergeWorkerOptionsDictionary( "contactArtistsFinal", // tableName "contactArtists", // prefix storageTimeout, // timeToLive "recommendation" // schemaName ); var mergeTaskId = await taskManager.RegisterDeferredTaskAsync( mergeOptions, // workerOptions new[] // prerequisiteTaskIds { projectionTaskId }, taskTimeout // expiresAfter ); _taskIds.Add(mergeTaskId); // Task for recommendation var recommendationOptions = new DeferredWorkerOptionsDictionary( typeof(AlbumRecommendationWorker).AssemblyQualifiedName, // workerType new Dictionary <string, string> // options { { AlbumRecommendationWorker.OptionSourceTableName, "contactArtistsFinal" }, { AlbumRecommendationWorker.OptionTargetTableName, "contactRecommendations" }, { AlbumRecommendationWorker.OptionSchemaName, "recommendation" }, { AlbumRecommendationWorker.OptionLimit, "5" } } ); var recommendationTaskId = await taskManager.RegisterDeferredTaskAsync( recommendationOptions, // workerOptions new[] // prerequisiteTaskIds { mergeTaskId }, taskTimeout // expiresAfter ); _taskIds.Add(recommendationTaskId); // Task to store facet var storeFacetOptions = new DeferredWorkerOptionsDictionary( typeof(RecommendationFacetStorageWorker).AssemblyQualifiedName, // workerType new Dictionary <string, string> // options { { RecommendationFacetStorageWorker.OptionTableName, "contactRecommendations" }, { RecommendationFacetStorageWorker.OptionSchemaName, "recommendation" } } ); var storeFacetTaskId = await taskManager.RegisterDeferredTaskAsync( storeFacetOptions, // workerOptions new[] // prerequisiteTaskIds { recommendationTaskId }, taskTimeout // expiresAfter ); _taskIds.Add(storeFacetTaskId); foreach (var taskId in _taskIds) { System.Console.WriteLine($"Registered task {taskId}"); } }
/// <summary> /// This the the core method that runs the whole nine yards for Projection, Merge and Store Recommendations /// </summary> /// <returns></returns> private async Task RegisterRecommendationTaskAsync() { var taskManager = GetTaskManager(); var xConnectClient = GetXConnectClient(); var taskTimeout = TimeSpan.FromMinutes(20); var storageTimeout = TimeSpan.FromMinutes(30); //This ID should be swapped with your custom GOAL ID, should be ideally read from a constants file var goalId = new Guid("{56367C18-B211-431B-A2C7-975F9C59372F}"); //Below will latest interactions on all contacts that belong to custom goal type. The time here depends on frequency of run of your processing service var query = xConnectClient.Contacts.Where(contact => contact.Interactions.Any(interaction => interaction.Events.OfType <Goal>().Any(x => x.DefinitionId == goalId) && interaction.EndDateTime > DateTime.UtcNow.AddMinutes(-120) ) ); var expandOptions = new ContactExpandOptions { Interactions = new RelatedInteractionsExpandOptions() }; query = query.WithExpandOptions(expandOptions); var searchRequest = query.GetSearchRequest(); // projection starts here var dataSourceOptions = new ContactSearchDataSourceOptionsDictionary( searchRequest, 30, 50 ); var projectionOptions = new ContactProjectionWorkerOptionsDictionary( typeof(MovieRecommendationModel).AssemblyQualifiedName, storageTimeout, "recommendation", new Dictionary <string, string> { { MovieRecommendationModel.OptionTableName, "contactMovies" } } ); var projectionTaskId = await taskManager.RegisterDistributedTaskAsync( dataSourceOptions, projectionOptions, null, taskTimeout ); //merge starts here var mergeOptions = new MergeWorkerOptionsDictionary( "contactMoviesFinal", // tableName "contactMovies", // prefix storageTimeout, // timeToLive "recommendation" // schemaName ); var mergeTaskId = await taskManager.RegisterDeferredTaskAsync( mergeOptions, // workerOptions new[] // prerequisiteTaskIds { projectionTaskId }, taskTimeout // expiresAfter ); var workerOptions = new DeferredWorkerOptionsDictionary( typeof(MovieRecommendationWorker).AssemblyQualifiedName, // workerType new Dictionary <string, string> // options { { MovieRecommendationWorker.OptionSourceTableName, "contactMoviesFinal" }, { MovieRecommendationWorker.OptionTargetTableName, "contactRecommendations" }, { MovieRecommendationWorker.OptionSchemaName, "recommendation" }, { MovieRecommendationWorker.OptionLimit, "20" } }); //recommendation task var recommendationTaskId = await taskManager.RegisterDeferredTaskAsync( workerOptions, // workerOptions new[] // prerequisiteTaskIds { mergeTaskId }, taskTimeout // expiresAfter ); //Facet storage var storageOptions = new DeferredWorkerOptionsDictionary( typeof(RecommendationFacetStorageWorker).AssemblyQualifiedName, new Dictionary <string, string> { { RecommendationFacetStorageWorker.OptionTableName, "contactRecommendations" }, { RecommendationFacetStorageWorker.OptionSchemaName, "recommendation" } }); var storageTask = await taskManager.RegisterDeferredTaskAsync( storageOptions, // workerOptions new[] // prerequisiteTaskIds { recommendationTaskId }, taskTimeout // expiresAfter ); }