public async Task Import(IJobCancellationToken token) { using var _ = _hub.PushScope(); var transaction = _hub.StartTransaction("import-catalog", "Import nuget catalog"); var logger = _loggerFactory.CreateLogger <NuGetCatalogImporter>(); logger.LogInformation("Starting importing catalog."); var httpClient = _httpClientFactory.CreateClient("nuget"); var catalogClient = new CatalogClient(httpClient, _loggerFactory.CreateLogger <CatalogClient>()); var settings = new CatalogProcessorSettings { DefaultMinCommitTimestamp = DateTimeOffset.MinValue, // Read everything ExcludeRedundantLeaves = false, }; var catalogProcessor = new CatalogProcessor( _cursorStore, catalogClient, _catalogLeafProcessor, settings, _loggerFactory.CreateLogger <CatalogProcessor>()); await catalogProcessor.ProcessAsync(token.ShutdownToken); logger.LogInformation("Finished importing catalog."); transaction.Finish(SpanStatus.Ok); }
public BatchCatalogProcessor( ICursor cursor, ICatalogClient client, ICatalogLeafProcessor leafProcessor, CatalogProcessorSettings settings, ILogger <BatchCatalogProcessor> logger) { _leafProcessor = leafProcessor ?? throw new ArgumentNullException(nameof(leafProcessor)); _client = client ?? throw new ArgumentNullException(nameof(client)); _cursor = cursor ?? throw new ArgumentNullException(nameof(cursor)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); if (settings == null) { throw new ArgumentNullException(nameof(settings)); } if (settings.ServiceIndexUrl == null) { throw new ArgumentException( $"The {nameof(CatalogProcessorSettings.ServiceIndexUrl)} property of the " + $"{nameof(CatalogProcessorSettings)} must not be null.", nameof(settings)); } // Clone the settings to avoid mutability issues. _settings = settings.Clone(); }
public async Task Import(IJobCancellationToken token) { using var _ = _hub.PushScope(); var transaction = _hub.StartTransaction("import-catalog", "catalog.import"); _hub.ConfigureScope(s => s.Transaction = transaction); var logger = _loggerFactory.CreateLogger <NuGetCatalogImporter>(); logger.LogInformation("Starting importing catalog."); try { var httpClient = _httpClientFactory.CreateClient("nuget"); var catalogClient = new CatalogClient(httpClient, _loggerFactory.CreateLogger <CatalogClient>()); var settings = new CatalogProcessorSettings { DefaultMinCommitTimestamp = DateTimeOffset.MinValue, // Read everything ExcludeRedundantLeaves = false, }; var catalogProcessor = new CatalogProcessor( _cursorStore, catalogClient, _catalogLeafProcessor, settings, _loggerFactory.CreateLogger <CatalogProcessor>()); await catalogProcessor.ProcessAsync(token.ShutdownToken); logger.LogInformation("Finished importing catalog."); transaction.Finish(SpanStatus.Ok); } catch (Exception e) { transaction.Finish(e); SentrySdk.CaptureException(e); throw; } finally { await SentrySdk.FlushAsync(TimeSpan.FromSeconds(2)); } }
public async override Task Run() { var token = new CancellationToken(); try { var regionInformations = _configuration.RegionInformations; var instances = new List <Instance>(); foreach (var regionInformation in regionInformations) { instances.AddRange(await _searchServiceClient.GetSearchEndpointsAsync(regionInformation, token)); } var maxCommit = DateTimeOffset.MinValue; foreach (Instance instance in instances) { try { var commitDateTime = await _searchServiceClient.GetCommitDateTimeAsync(instance, token); maxCommit = commitDateTime > maxCommit ? commitDateTime : maxCommit; } catch (Exception e) { Logger.LogError("An exception was encountered so no HTTP response was returned. {Exception}", e); } } if (maxCommit == DateTimeOffset.MinValue) { Logger.LogError("Failed to retrieve a proper starting commit. Abandoning the current run."); return; } var catalogLeafProcessor = new PackageLagCatalogLeafProcessor(instances, _searchServiceClient, _telemetryService, LoggerFactory.CreateLogger <PackageLagCatalogLeafProcessor>()); if (_configuration.RetryLimit > 0) { catalogLeafProcessor.RetryLimit = _configuration.RetryLimit; } if (_configuration.WaitBetweenRetrySeconds > 0) { catalogLeafProcessor.WaitBetweenPolls = TimeSpan.FromSeconds(_configuration.WaitBetweenRetrySeconds); } var settings = new CatalogProcessorSettings { ServiceIndexUrl = _configuration.ServiceIndexUrl, DefaultMinCommitTimestamp = maxCommit, ExcludeRedundantLeaves = false }; var start = new FileCursor("cursor.json", LoggerFactory.CreateLogger <FileCursor>()); await start.SetAsync(maxCommit.AddTicks(1)); var catalogProcessor = new CatalogProcessor(start, _catalogClient, catalogLeafProcessor, settings, LoggerFactory.CreateLogger <CatalogProcessor>()); bool success; int retryCount = 0; do { success = await catalogProcessor.ProcessAsync(); if (!success || !await catalogLeafProcessor.WaitForProcessing()) { retryCount++; Logger.LogError("Processing the catalog leafs failed. Retry Count {CatalogProcessRetryCount}", retryCount); } }while (!success && retryCount < MAX_CATALOG_RETRY_COUNT); return; } catch (Exception e) { Logger.LogError("Exception Occured. {Exception}", e); return; } }