public void Initialize(IServiceCollection serviceCollection)
{
var snapshot = serviceCollection.BuildServiceProvider();
var settingsManager = snapshot.GetService <ISettingsManager>();
serviceCollection.AddSingleton <ISearchPhraseParser, SearchPhraseParser>();
// Allow scale out of indexation through background worker, if opted-in.
if (settingsManager.GetValue(ModuleConstants.Settings.General.IndexingJobs.ScaleOut.Name, false))
{
serviceCollection.AddSingleton <IIndexingWorker>(new HangfireIndexingWorker
{
ThrottleQueueCount = settingsManager.GetValue(ModuleConstants.Settings.General.IndexingJobs.MaxQueueSize.Name, 25)
});
}
else
{
serviceCollection.AddSingleton <IIndexingWorker>(c => null);
}
serviceCollection.AddSingleton <IIndexingManager, IndexingManager>();
//TODO delete it after implementation in the modules
var productIndexingConfiguration = new IndexDocumentConfiguration
{
DocumentType = KnownDocumentTypes.Product,
DocumentSource = new IndexDocumentSource()
};
serviceCollection.AddSingleton(new[] { productIndexingConfiguration });
}
private async Task <BatchIndexingOptions> GetBatchOptionsAsync(IndexDocumentConfiguration configuration, IndexingOptions options)
{
var result = new BatchIndexingOptions
{
DocumentType = options.DocumentType,
DocumentIds = options.DocumentIds,
StartDate = options.StartDate,
EndDate = options.EndDate,
BatchSize = options.BatchSize,
PrimaryDocumentBuilder = configuration.DocumentSource.DocumentBuilder,
SecondaryDocumentBuilders = configuration.RelatedSources?.Where(s => s.DocumentBuilder != null).Select(s => s.DocumentBuilder).ToList(),
};
if (options.DocumentIds != null)
{
result.TotalCount = options.DocumentIds.Count;
}
else
{
result.ChangesProvidersAndTotalCounts = await GetChangesProvidersAndTotalCountsAsync(configuration, options.StartDate, options.EndDate);
result.TotalCount = result.ChangesProvidersAndTotalCounts.Sum(p => p.TotalCount);
}
return(result);
}
public void Initialize(IServiceCollection serviceCollection)
{
serviceCollection.AddSingleton <ISearchPhraseParser, SearchPhraseParser>();
serviceCollection.AddSingleton <IIndexingWorker>(context =>
{
var settingsManager = context.GetService <ISettingsManager>();
if (settingsManager.GetValue(ModuleConstants.Settings.IndexingJobs.ScaleOut.Name, false))
{
return(new HangfireIndexingWorker
{
ThrottleQueueCount = settingsManager.GetValue(ModuleConstants.Settings.IndexingJobs.MaxQueueSize.Name, 25)
});
}
else
{
return(null);
}
});
serviceCollection.AddSingleton <IIndexingManager, IndexingManager>();
//TODO delete it after implementation in the modules
var productIndexingConfiguration = new IndexDocumentConfiguration
{
DocumentType = KnownDocumentTypes.Product,
DocumentSource = new IndexDocumentSource()
};
serviceCollection.AddSingleton(new[] { productIndexingConfiguration });
}
public void Initialize(IServiceCollection serviceCollection)
{
var configuration = serviceCollection.BuildServiceProvider().GetRequiredService <IConfiguration>();
serviceCollection.AddTransient <ICustomerRepository, CustomerRepositoryImpl>();
var connectionString = configuration.GetConnectionString("VirtoCommerce.Customer") ?? configuration.GetConnectionString("VirtoCommerce");
serviceCollection.AddDbContext <CustomerDbContext>(options => options.UseSqlServer(connectionString));
serviceCollection.AddSingleton <Func <ICustomerRepository> >(provider => () => provider.CreateScope().ServiceProvider.GetRequiredService <ICustomerRepository>());
serviceCollection.AddSingleton <Func <IMemberRepository> >(provider => () => provider.CreateScope().ServiceProvider.GetRequiredService <ICustomerRepository>());
serviceCollection.AddSingleton <IMemberService, CommerceMembersServiceImpl>();
serviceCollection.AddSingleton <ISearchRequestBuilder, MemberSearchRequestBuilder>();
serviceCollection.AddSingleton <MemberSearchServiceBase>();
serviceCollection.AddSingleton <MemberIndexedSearchService>();
serviceCollection.AddSingleton <CommerceMembersSearchServiceImpl>();
serviceCollection.AddSingleton <IMemberSearchService, MemberSearchServiceDecorator>();
serviceCollection.AddSingleton <CustomerExportImport>();
var snapshot = serviceCollection.BuildServiceProvider();
var memberIndexingConfiguration = new IndexDocumentConfiguration
{
DocumentType = KnownDocumentTypes.Member,
DocumentSource = new IndexDocumentSource
{
ChangesProvider = snapshot.GetService <MemberDocumentChangesProvider>(),
DocumentBuilder = snapshot.GetService <MemberDocumentBuilder>(),
},
};
serviceCollection.AddSingleton(memberIndexingConfiguration);
serviceCollection.AddSingleton <MemberChangedEventHandler>();
}
public override void PostInitialize()
{
base.PostInitialize();
var securityScopeService = _container.Resolve <IPermissionScopeService>();
securityScopeService.RegisterSope(() => new CatalogSelectedScope());
securityScopeService.RegisterSope(() => new CatalogSelectedCategoryScope(_container.Resolve <ICategoryService>()));
var httpConfiguration = _container.Resolve <HttpConfiguration>();
httpConfiguration.Formatters.JsonFormatter.SerializerSettings.Converters.Add(new ProductSearchCriteriaJsonConverter());
httpConfiguration.Formatters.JsonFormatter.SerializerSettings.Converters.Add(new CategorySearchCriteriaJsonConverter());
// Register dynamic property for storing browsing filters
var filteredBrowsingProperty = new DynamicProperty
{
Id = "2b15f370ab524186bec1ace82509a60a",
Name = BrowseFilterService.FilteredBrowsingPropertyName,
ObjectType = typeof(Store).FullName,
ValueType = DynamicPropertyValueType.LongText,
CreatedBy = "Auto"
};
var dynamicPropertyService = _container.Resolve <IDynamicPropertyService>();
dynamicPropertyService.SaveProperties(new[] { filteredBrowsingProperty });
// Product indexing configuration
var productIndexingConfiguration = new IndexDocumentConfiguration
{
DocumentType = KnownDocumentTypes.Product,
DocumentSource = new IndexDocumentSource
{
ChangesProvider = _container.Resolve <ProductDocumentChangesProvider>(),
DocumentBuilder = _container.Resolve <ProductDocumentBuilder>(),
},
};
_container.RegisterInstance(productIndexingConfiguration.DocumentType, productIndexingConfiguration);
// Category indexing configuration
var categoryIndexingConfiguration = new IndexDocumentConfiguration
{
DocumentType = KnownDocumentTypes.Category,
DocumentSource = new IndexDocumentSource
{
ChangesProvider = _container.Resolve <CategoryDocumentChangesProvider>(),
DocumentBuilder = _container.Resolve <CategoryDocumentBuilder>(),
},
};
_container.RegisterInstance(categoryIndexingConfiguration.DocumentType, categoryIndexingConfiguration);
}
protected virtual async Task ProcessConfigurationAsync(IndexDocumentConfiguration configuration, IndexingOptions options, Action <IndexingProgress> progressCallback, CancellationToken cancellationToken)
{
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
if (string.IsNullOrEmpty(configuration.DocumentType))
{
throw new ArgumentNullException($"{nameof(configuration)}.{nameof(configuration.DocumentType)}");
}
if (configuration.DocumentSource == null)
{
throw new ArgumentNullException($"{nameof(configuration)}.{nameof(configuration.DocumentSource)}");
}
if (configuration.DocumentSource.ChangesProvider == null)
{
throw new ArgumentNullException($"{nameof(configuration)}.{nameof(configuration.DocumentSource)}.{nameof(configuration.DocumentSource.ChangesProvider)}");
}
if (configuration.DocumentSource.DocumentBuilder == null)
{
throw new ArgumentNullException($"{nameof(configuration)}.{nameof(configuration.DocumentSource)}.{nameof(configuration.DocumentSource.DocumentBuilder)}");
}
cancellationToken.ThrowIfCancellationRequested();
var documentType = options.DocumentType;
progressCallback?.Invoke(new IndexingProgress($"{documentType}: calculating total count", documentType));
var batchOptions = await GetBatchOptionsAsync(configuration, options);
var processedCount = 0L;
var totalCount = batchOptions.TotalCount;
while (processedCount < totalCount)
{
var batchResult = await ProcessBatchAsync(batchOptions, cancellationToken);
// Avoid infinite loop
if (batchResult.ProcessedCount < 1)
{
break;
}
processedCount += batchResult.ProcessedCount;
batchOptions.Skip += batchOptions.BatchSize;
var errors = GetIndexingErrors(batchResult.IndexingResult);
progressCallback?.Invoke(new IndexingProgress($"{documentType}: {processedCount} of {totalCount} have been indexed", documentType, totalCount, processedCount, errors));
}
progressCallback?.Invoke(new IndexingProgress($"{documentType}: indexation finished", documentType, totalCount, processedCount));
}
private static IIndexingManager GetIndexingManager(ISearchProvider searchProvider, IList <DocumentSource> documentSources)
{
var primaryDocumentSource = documentSources?.FirstOrDefault();
var configuration = new IndexDocumentConfiguration
{
DocumentType = DocumentType,
DocumentSource = CreateIndexDocumentSource(primaryDocumentSource),
RelatedSources = documentSources?.Skip(1).Select(CreateIndexDocumentSource).ToArray(),
};
return(new IndexingManager(searchProvider, new[] { configuration }, new Moq.Mock <ISearchConnection>().Object));
}
protected virtual async Task <IIndexDocumentChangeFeed[]> GetChangeFeeds(
IndexDocumentConfiguration configuration, IndexingOptions options)
{
// Return in-memory change feed for specific set of document ids.
if (options.DocumentIds != null)
{
return(new IIndexDocumentChangeFeed[]
{
new InMemoryIndexDocumentChangeFeed(options.DocumentIds.ToArray(),
IndexDocumentChangeType.Modified, options.BatchSize ?? 50)
});
}
// Support old ChangesProvider.
if (configuration.DocumentSource.ChangeFeedFactory == null)
{
configuration.DocumentSource.ChangeFeedFactory =
new IndexDocumentChangeFeedFactoryAdapter(configuration.DocumentSource.ChangesProvider);
}
var factories = new List <IIndexDocumentChangeFeedFactory>
{
configuration.DocumentSource.ChangeFeedFactory
};
// In case of 'full' re-index we don't want to include the related sources,
// because that would double the indexation work.
// E.g. All products would get indexed for the primary document source
// and afterwards all products would get re-indexed for all the prices as well.
if (options.StartDate != null || options.EndDate != null)
{
foreach (var related in configuration.RelatedSources ?? Enumerable.Empty <IndexDocumentSource>())
{
// Support old ChangesProvider.
if (related.ChangeFeedFactory == null)
{
related.ChangeFeedFactory = new IndexDocumentChangeFeedFactoryAdapter(related.ChangesProvider);
}
factories.Add(related.ChangeFeedFactory);
}
}
return(await Task.WhenAll(factories.Select(x =>
x.CreateFeed(options.StartDate, options.EndDate, options.BatchSize ?? 50))));
}
public override void PostInitialize()
{
base.PostInitialize();
AbstractTypeFactory <Member> .RegisterType <Organization>().MapToType <OrganizationDataEntity>();
AbstractTypeFactory <Member> .RegisterType <Contact>().MapToType <ContactDataEntity>();
AbstractTypeFactory <Member> .RegisterType <Vendor>().MapToType <VendorDataEntity>();
AbstractTypeFactory <Member> .RegisterType <Employee>().MapToType <EmployeeDataEntity>();
AbstractTypeFactory <MemberDataEntity> .RegisterType <ContactDataEntity>();
AbstractTypeFactory <MemberDataEntity> .RegisterType <OrganizationDataEntity>();
AbstractTypeFactory <MemberDataEntity> .RegisterType <VendorDataEntity>();
AbstractTypeFactory <MemberDataEntity> .RegisterType <EmployeeDataEntity>();
//Next lines allow to use polymorph types in API controller methods
var httpConfiguration = _container.Resolve <HttpConfiguration>();
httpConfiguration.Formatters.JsonFormatter.SerializerSettings.Converters.Add(new PolymorphicMemberJsonConverter());
// Indexing configuration
var memberIndexingConfiguration = new IndexDocumentConfiguration
{
DocumentType = KnownDocumentTypes.Member,
DocumentSource = new IndexDocumentSource
{
ChangesProvider = _container.Resolve <MemberDocumentChangesProvider>(),
DocumentBuilder = _container.Resolve <MemberDocumentBuilder>(),
},
};
_container.RegisterInstance(memberIndexingConfiguration.DocumentType, memberIndexingConfiguration);
}
protected virtual async Task ProcessConfigurationAsync(IndexDocumentConfiguration configuration, IndexingOptions options, Action <IndexingProgress> progressCallback, ICancellationToken cancellationToken)
{
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
if (string.IsNullOrEmpty(configuration.DocumentType))
{
throw new ArgumentNullException($"{nameof(configuration)}.{nameof(configuration.DocumentType)}");
}
if (configuration.DocumentSource == null)
{
throw new ArgumentNullException($"{nameof(configuration)}.{nameof(configuration.DocumentSource)}");
}
if (configuration.DocumentSource.ChangesProvider == null)
{
throw new ArgumentNullException($"{nameof(configuration)}.{nameof(configuration.DocumentSource)}.{nameof(configuration.DocumentSource.ChangesProvider)}");
}
if (configuration.DocumentSource.DocumentBuilder == null)
{
throw new ArgumentNullException($"{nameof(configuration)}.{nameof(configuration.DocumentSource)}.{nameof(configuration.DocumentSource.DocumentBuilder)}");
}
cancellationToken.ThrowIfCancellationRequested();
var documentType = options.DocumentType;
progressCallback?.Invoke(new IndexingProgress($"{documentType}: calculating total count", documentType));
var batchOptions = new BatchIndexingOptions
{
DocumentType = options.DocumentType,
PrimaryDocumentBuilder = configuration.DocumentSource.DocumentBuilder,
SecondaryDocumentBuilders = configuration.RelatedSources
?.Where(s => s.DocumentBuilder != null)
.Select(s => s.DocumentBuilder)
.ToList(),
};
var feeds = await GetChangeFeeds(configuration, options);
// Try to get total count to indicate progress. Some feeds don't have a total count.
var totalCount = feeds.Any(x => x.TotalCount == null)
? (long?)null
: feeds.Sum(x => x.TotalCount ?? 0);
long processedCount = 0;
var changes = await GetNextChangesAsync(feeds);
while (changes.Any())
{
IList <string> errors = null;
if (_backgroundWorker == null)
{
var indexingResult = await ProcessChangesAsync(changes, batchOptions, cancellationToken);
errors = GetIndexingErrors(indexingResult);
}
else
{
// We're executing a job to index all documents or the changes since a specific time.
// Priority for this indexation work should be quite low.
var documentIds = changes
.Select(x => x.DocumentId)
.Distinct()
.ToArray();
_backgroundWorker.IndexDocuments(configuration.DocumentType, documentIds, IndexingPriority.Background);
}
processedCount += changes.Count;
var description = totalCount != null
? $"{documentType}: {processedCount} of {totalCount} have been indexed"
: $"{documentType}: {processedCount} have been indexed";
progressCallback?.Invoke(new IndexingProgress(description, documentType, totalCount, processedCount, errors));
changes = await GetNextChangesAsync(feeds);
}
progressCallback?.Invoke(new IndexingProgress($"{documentType}: indexation finished", documentType, totalCount ?? processedCount, processedCount));
}
public override void PostInitialize()
{
base.PostInitialize();
var securityScopeService = _container.Resolve <IPermissionScopeService>();
securityScopeService.RegisterSope(() => new CatalogSelectedScope());
securityScopeService.RegisterSope(() => new CatalogSelectedCategoryScope(_container.Resolve <ICategoryService>()));
var httpConfiguration = _container.Resolve <HttpConfiguration>();
httpConfiguration.Formatters.JsonFormatter.SerializerSettings.Converters.Add(new SearchCriteriaJsonConverter());
// Register dynamic property for storing browsing filters
var filteredBrowsingProperty = new DynamicProperty
{
Id = "2b15f370ab524186bec1ace82509a60a",
Name = BrowseFilterService.FilteredBrowsingPropertyName,
ObjectType = typeof(Store).FullName,
ValueType = DynamicPropertyValueType.LongText,
CreatedBy = "Auto"
};
var dynamicPropertyService = _container.Resolve <IDynamicPropertyService>();
dynamicPropertyService.SaveProperties(new[] { filteredBrowsingProperty });
// Product indexing configuration
var productIndexingConfiguration = new IndexDocumentConfiguration
{
DocumentType = KnownDocumentTypes.Product,
DocumentSource = new IndexDocumentSource
{
ChangesProvider = _container.Resolve <ProductDocumentChangesProvider>(),
DocumentBuilder = _container.Resolve <ProductDocumentBuilder>(),
},
};
_container.RegisterInstance(productIndexingConfiguration.DocumentType, productIndexingConfiguration);
// Category indexing configuration
var categoryIndexingConfiguration = new IndexDocumentConfiguration
{
DocumentType = KnownDocumentTypes.Category,
DocumentSource = new IndexDocumentSource
{
ChangesProvider = _container.Resolve <CategoryDocumentChangesProvider>(),
DocumentBuilder = _container.Resolve <CategoryDocumentBuilder>(),
},
};
_container.RegisterInstance(categoryIndexingConfiguration.DocumentType, categoryIndexingConfiguration);
#region Register types for generic Export
var registrar = _container.Resolve <IKnownExportTypesRegistrar>();
registrar.RegisterType(
ExportedTypeDefinitionBuilder.Build <ExportableCatalogFull, CatalogFullExportDataQuery>()
.WithDataSourceFactory(_container.Resolve <ICatalogExportPagedDataSourceFactory>())
.WithPermissionAuthorization(CatalogPredefinedPermissions.Export, CatalogPredefinedPermissions.Read)
.WithMetadata(new ExportedTypeMetadata {
PropertyInfos = Array.Empty <ExportedTypePropertyInfo>()
})
.WithAuthorizationHandler(_container.Resolve <CatalogExportSecurityHandler>())
);
registrar.RegisterType(
ExportedTypeDefinitionBuilder.Build <ExportableProduct, ProductExportDataQuery>()
.WithDataSourceFactory(_container.Resolve <ICatalogExportPagedDataSourceFactory>())
.WithMetadata(typeof(ExportableProduct).GetPropertyNames(
nameof(ExportableProduct.Properties),
$"{nameof(ExportableProduct.Properties)}.{nameof(Property.Attributes)}",
$"{nameof(ExportableProduct.Properties)}.{nameof(Property.DisplayNames)}",
$"{nameof(ExportableProduct.Properties)}.{nameof(Property.ValidationRules)}",
nameof(ExportableProduct.PropertyValues),
nameof(ExportableProduct.Assets),
nameof(ExportableProduct.Links),
nameof(ExportableProduct.SeoInfos),
nameof(ExportableProduct.Reviews),
nameof(ExportableProduct.Associations),
nameof(ExportableProduct.ReferencedAssociations),
nameof(ExportableProduct.Outlines),
nameof(ExportableProduct.Images)))
.WithTabularMetadata(typeof(ExportableProduct).GetPropertyNames())
.WithPermissionAuthorization(CatalogPredefinedPermissions.Export, CatalogPredefinedPermissions.Read)
.WithAuthorizationHandler(_container.Resolve <CatalogExportSecurityHandler>()));
#endregion
}
protected virtual async Task <ChangesProviderAndTotalCount[]> GetChangesProvidersAndTotalCountsAsync(IndexDocumentConfiguration configuration, DateTime?startDate, DateTime?endDate)
{
var changesProviders = new List <IIndexDocumentChangesProvider> {
configuration.DocumentSource.ChangesProvider
};
if (configuration.RelatedSources != null)
{
changesProviders.AddRange(configuration.RelatedSources.Where(s => s.ChangesProvider != null).Select(s => s.ChangesProvider));
}
var result = await Task.WhenAll(changesProviders.Select(async p => new ChangesProviderAndTotalCount {
Provider = p, TotalCount = await p.GetTotalChangesCountAsync(startDate, endDate)
}));
return(result);
}
