public CatalogLeafFetcher(
IRegistrationClient registrationClient,
ICatalogClient catalogClient,
IOptionsSnapshot <Catalog2AzureSearchConfiguration> options,
IAzureSearchTelemetryService telemetryService,
ILogger <CatalogLeafFetcher> logger)
{
_registrationClient = registrationClient ?? throw new ArgumentNullException(nameof(registrationClient));
_catalogClient = catalogClient ?? throw new ArgumentNullException(nameof(catalogClient));
_options = options ?? throw new ArgumentNullException(nameof(options));
_telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (_options.Value.MaxConcurrentBatches <= 0)
{
throw new ArgumentOutOfRangeException(
nameof(options),
$"The {nameof(AzureSearchJobConfiguration.MaxConcurrentBatches)} must be greater than zero.");
}
if (_options.Value.RegistrationsBaseUrl == null)
{
throw new ArgumentException(
$"The {nameof(Catalog2AzureSearchConfiguration.RegistrationsBaseUrl)} must be set.",
nameof(options));
}
}
public CatalogProcessor(
ICursor cursor,
ICatalogClient client,
ICatalogLeafProcessor leafProcessor,
CatalogProcessorSettings settings,
ILogger <CatalogProcessor> logger)
{
_leafProcessor = leafProcessor ?? throw new ArgumentNullException(nameof(leafProcessor));
_client = client ?? throw new ArgumentNullException(nameof(client));
_cursor = cursor ?? throw new ArgumentNullException(nameof(cursor));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (settings.ServiceIndexUrl == null)
{
throw new ArgumentException(
$"The {nameof(CatalogProcessorSettings.ServiceIndexUrl)} property of the " +
$"{nameof(CatalogProcessorSettings)} must not be null.",
nameof(settings));
}
// Clone the settings to avoid mutability issues.
_settings = settings.Clone();
}
private async Task <IEnumerable <CatalogLeafItem> > GetCatalogLeafItems(
ICatalogClient catalogClient,
CatalogIndex catalogIndex,
DateTimeOffset cursor,
CancellationToken cancellationToken)
{
var catalogLeafItems = new ConcurrentBag <CatalogLeafItem>();
var catalogPageUrls = new ConcurrentBag <string>(
catalogIndex
.Items
.Where(i => i.CommitTimestamp > cursor)
.Select(i => i.CatalogPageUrl));
await ParallelHelper.ProcessInParallel(
catalogPageUrls,
ProcessCatalogPageUrlAsync,
cancellationToken);
return(catalogLeafItems);
async Task ProcessCatalogPageUrlAsync(string catalogPageUrl, CancellationToken token)
{
_logger.LogInformation("Processing catalog page {CatalogPageUrl}...", catalogPageUrl);
var page = await catalogClient.GetPageAsync(catalogPageUrl, token);
foreach (var catalogLeafItem in page.Items.Where(i => i.CommitTimestamp > cursor))
{
catalogLeafItems.Add(catalogLeafItem);
}
_logger.LogInformation("Processed catalog page {CatalogPageUrl}", catalogPageUrl);
}
}
public override void Init(IServiceContainer serviceContainer, IDictionary <string, string> jobArgsDictionary)
{
base.Init(serviceContainer, jobArgsDictionary);
_configuration = _serviceProvider.GetService <PackageLagMonitorConfiguration>();
_catalogClient = _serviceProvider.GetService <ICatalogClient>();
_searchServiceClient = _serviceProvider.GetService <ISearchServiceClient>();
_telemetryService = _serviceProvider.GetService <IPackageLagTelemetryService>();
}
public JsonStrike(
ICatalogClient catalogClient,
IRegistrationUpdater updater,
string packageId)
{
_catalogClient = catalogClient;
_updater = updater;
_packageId = packageId;
_urls = new List <string>();
}
public override void Init(IServiceContainer serviceContainer, IDictionary <string, string> jobArgsDictionary)
{
var configurationFilename = JobConfigurationManager.GetArgument(jobArgsDictionary, ConfigurationArgument);
_serviceProvider = GetServiceProvider(GetConfigurationRoot(configurationFilename));
_configuration = _serviceProvider.GetService <PackageLagMonitorConfiguration>();
_catalogClient = _serviceProvider.GetService <ICatalogClient>();
_httpClient = _serviceProvider.GetService <IHttpClientWrapper>();
_searchServiceClient = _serviceProvider.GetService <ISearchServiceClient>();
_telemetryService = _serviceProvider.GetService <IPackageLagTelemetryService>();
}
/// <summary>
/// Create a processor to discover and download catalog leafs. Leafs are processed
/// by the <see cref="ICatalogLeafProcessor"/>.
/// </summary>
/// <param name="cursor">Cursor to track succesfully processed leafs. Leafs before the cursor are skipped.</param>
/// <param name="client">The client to interact with the catalog resource.</param>
/// <param name="leafProcessor">The leaf processor.</param>
/// <param name="options">The options to configure catalog processing.</param>
/// <param name="logger">The logger used for telemetry.</param>
public CatalogProcessor(
ICursor cursor,
ICatalogClient client,
ICatalogLeafProcessor leafProcessor,
CatalogProcessorOptions options,
ILogger <CatalogProcessor> logger)
{
_leafProcessor = leafProcessor ?? throw new ArgumentNullException(nameof(leafProcessor));
_client = client ?? throw new ArgumentNullException(nameof(client));
_cursor = cursor ?? throw new ArgumentNullException(nameof(cursor));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public CommitCollectorUtility(
ICatalogClient catalogClient,
IV3TelemetryService telemetryService,
IOptionsSnapshot <CommitCollectorConfiguration> options,
ILogger <CommitCollectorUtility> logger)
{
_catalogClient = catalogClient ?? throw new ArgumentNullException(nameof(catalogClient));
_telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (_options.Value.MaxConcurrentCatalogLeafDownloads <= 0)
{
throw new ArgumentOutOfRangeException(
nameof(options),
$"The {nameof(ICommitCollectorConfiguration.MaxConcurrentCatalogLeafDownloads)} must be greater than zero.");
}
}
public IconsCollector(
Uri index,
ITelemetryService telemetryService,
IStorageFactory targetStorageFactory,
ICatalogClient catalogClient,
ICatalogLeafDataProcessor catalogLeafDataProcessor,
IIconCopyResultCachePersistence iconCopyResultCache,
IStorageFactory iconCacheStorageFactory,
Func <HttpMessageHandler> httpHandlerFactory,
ILogger <IconsCollector> logger)
: base(index, telemetryService, httpHandlerFactory, httpClientTimeout: TimeSpan.FromMinutes(5))
{
_targetStorageFactory = targetStorageFactory ?? throw new ArgumentNullException(nameof(targetStorageFactory));
_catalogClient = catalogClient ?? throw new ArgumentNullException(nameof(catalogClient));
_catalogLeafDataProcessor = catalogLeafDataProcessor ?? throw new ArgumentNullException(nameof(catalogLeafDataProcessor));
_iconCopyResultCache = iconCopyResultCache ?? throw new ArgumentNullException(nameof(iconCopyResultCache));
_iconCacheStorageFactory = iconCacheStorageFactory ?? throw new ArgumentNullException(nameof(iconCacheStorageFactory));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public Db2AzureSearchCommand(
INewPackageRegistrationProducer producer,
IPackageEntityIndexActionBuilder indexActionBuilder,
IBlobContainerBuilder blobContainerBuilder,
IIndexBuilder indexBuilder,
Func <IBatchPusher> batchPusherFactory,
ICatalogClient catalogClient,
IStorageFactory storageFactory,
IOwnerDataClient ownerDataClient,
IDownloadDataClient downloadDataClient,
IVerifiedPackagesDataClient verifiedPackagesDataClient,
IPopularityTransferDataClient popularityTransferDataClient,
IOptionsSnapshot <Db2AzureSearchConfiguration> options,
IOptionsSnapshot <Db2AzureSearchDevelopmentConfiguration> developmentOptions,
ILogger <Db2AzureSearchCommand> logger)
{
_producer = producer ?? throw new ArgumentNullException(nameof(producer));
_indexActionBuilder = indexActionBuilder ?? throw new ArgumentNullException(nameof(indexActionBuilder));
_blobContainerBuilder = blobContainerBuilder ?? throw new ArgumentNullException(nameof(blobContainerBuilder));
_indexBuilder = indexBuilder ?? throw new ArgumentNullException(nameof(indexBuilder));
_batchPusherFactory = batchPusherFactory ?? throw new ArgumentNullException(nameof(batchPusherFactory));
_catalogClient = catalogClient ?? throw new ArgumentNullException(nameof(catalogClient));
_storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory));
_ownerDataClient = ownerDataClient ?? throw new ArgumentNullException(nameof(ownerDataClient));
_downloadDataClient = downloadDataClient ?? throw new ArgumentNullException(nameof(downloadDataClient));
_verifiedPackagesDataClient = verifiedPackagesDataClient ?? throw new ArgumentNullException(nameof(verifiedPackagesDataClient));
_popularityTransferDataClient = popularityTransferDataClient ?? throw new ArgumentNullException(nameof(popularityTransferDataClient));
_options = options ?? throw new ArgumentNullException(nameof(options));
_developmentOptions = developmentOptions ?? throw new ArgumentNullException(nameof(developmentOptions));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (_options.Value.MaxConcurrentBatches <= 0)
{
throw new ArgumentOutOfRangeException(
nameof(options),
$"The {nameof(AzureSearchJobConfiguration.MaxConcurrentBatches)} must be greater than zero.");
}
}
public async static Task <(CatalogIndex, IEnumerable <CatalogLeafItem>)> LoadCatalogAsync(
this ICatalogClient catalogClient,
DateTimeOffset minCursor,
DateTimeOffset maxCursor,
ILogger logger,
CancellationToken cancellationToken)
{
var catalogIndex = await catalogClient.GetIndexAsync(cancellationToken);
var catalogLeafItems = new ConcurrentBag <CatalogLeafItem>();
var catalogPageUrls = new ConcurrentBag <CatalogPageItem>(
catalogIndex.GetPagesInBounds(minCursor, maxCursor));
await ParallelAsync.RunAsync(
catalogPageUrls,
ProcessCatalogPageAsync,
cancellationToken);
return(catalogIndex, catalogLeafItems);
async Task ProcessCatalogPageAsync(CatalogPageItem pageItem, CancellationToken token)
{
logger.LogInformation("Processing catalog page {CatalogPageUrl}...", pageItem.CatalogPageUrl);
var page = await catalogClient.GetPageAsync(pageItem.CatalogPageUrl, token);
var leafs = page.GetLeavesInBounds(minCursor, maxCursor, excludeRedundantLeaves: true);
foreach (var catalogLeafItem in leafs)
{
catalogLeafItems.Add(catalogLeafItem);
}
logger.LogInformation("Processed catalog page {CatalogPageUrl}", pageItem.CatalogPageUrl);
}
}
public ReportingCommandExecutor(ICatalogClient catalogClient, IDataClient dataClient)
{
_catalogClient = catalogClient;
_dataClient = dataClient;
}
public static void AddAfterSelf(this JProperty property, ICatalogClient catalogClient, CatalogType catalogType, string propertyName)
{
var resultQuery = catalogClient.GetValue(catalogType, property.Value.ToString());
property.AddAfterSelf(new JProperty(propertyName, resultQuery.DESCRIPCION));
}
public BasketController(ICatalogClient catalogClient, IBasketClient basketClient, IConfiguration config)
{
_catalogClient = catalogClient;
_basketClient = basketClient;
_userName = config["userName"] ?? "defaultUser";
}
public CfdiLoader(ICatalogClient catalogClient)
{
_catalogClient = catalogClient;
}
public static void AddAfterSelf(this JProperty property, CatalogType catalogType, ICatalogClient catalogClient)
{
var resultQuery = catalogClient.GetValue(catalogType, property.Value.ToString());
property.AddAfterSelf(new JProperty($"{property.Name}Descripcion", resultQuery.DESCRIPCION));
}
public HomeController(ICatalogClient catalogClient)
{
_catalogClient = catalogClient;
}
public CatalogService(ICatalogClient catalogClient)
{
_catalogClient = catalogClient;
}
