public async Task Import(IJobCancellationToken token)
{
using var _ = _hub.PushScope();
var transaction = _hub.StartTransaction("import-catalog", "Import nuget catalog");
var logger = _loggerFactory.CreateLogger <NuGetCatalogImporter>();
logger.LogInformation("Starting importing catalog.");
var httpClient = _httpClientFactory.CreateClient("nuget");
var catalogClient = new CatalogClient(httpClient, _loggerFactory.CreateLogger <CatalogClient>());
var settings = new CatalogProcessorSettings
{
DefaultMinCommitTimestamp = DateTimeOffset.MinValue, // Read everything
ExcludeRedundantLeaves = false,
};
var catalogProcessor = new CatalogProcessor(
_cursorStore,
catalogClient,
_catalogLeafProcessor,
settings,
_loggerFactory.CreateLogger <CatalogProcessor>());
await catalogProcessor.ProcessAsync(token.ShutdownToken);
logger.LogInformation("Finished importing catalog.");
transaction.Finish(SpanStatus.Ok);
}
public BatchCatalogProcessor(
ICursor cursor,
ICatalogClient client,
ICatalogLeafProcessor leafProcessor,
CatalogProcessorSettings settings,
ILogger <BatchCatalogProcessor> logger)
{
_leafProcessor = leafProcessor ?? throw new ArgumentNullException(nameof(leafProcessor));
_client = client ?? throw new ArgumentNullException(nameof(client));
_cursor = cursor ?? throw new ArgumentNullException(nameof(cursor));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (settings.ServiceIndexUrl == null)
{
throw new ArgumentException(
$"The {nameof(CatalogProcessorSettings.ServiceIndexUrl)} property of the " +
$"{nameof(CatalogProcessorSettings)} must not be null.",
nameof(settings));
}
// Clone the settings to avoid mutability issues.
_settings = settings.Clone();
}
public async Task Import(IJobCancellationToken token)
{
using var _ = _hub.PushScope();
var transaction = _hub.StartTransaction("import-catalog", "catalog.import");
_hub.ConfigureScope(s => s.Transaction = transaction);
var logger = _loggerFactory.CreateLogger <NuGetCatalogImporter>();
logger.LogInformation("Starting importing catalog.");
try
{
var httpClient = _httpClientFactory.CreateClient("nuget");
var catalogClient = new CatalogClient(httpClient, _loggerFactory.CreateLogger <CatalogClient>());
var settings = new CatalogProcessorSettings
{
DefaultMinCommitTimestamp = DateTimeOffset.MinValue, // Read everything
ExcludeRedundantLeaves = false,
};
var catalogProcessor = new CatalogProcessor(
_cursorStore,
catalogClient,
_catalogLeafProcessor,
settings,
_loggerFactory.CreateLogger <CatalogProcessor>());
await catalogProcessor.ProcessAsync(token.ShutdownToken);
logger.LogInformation("Finished importing catalog.");
transaction.Finish(SpanStatus.Ok);
}
catch (Exception e)
{
transaction.Finish(e);
SentrySdk.CaptureException(e);
throw;
}
finally
{
await SentrySdk.FlushAsync(TimeSpan.FromSeconds(2));
}
}
public async override Task Run()
{
var token = new CancellationToken();
try
{
var regionInformations = _configuration.RegionInformations;
var instances = new List <Instance>();
foreach (var regionInformation in regionInformations)
{
instances.AddRange(await _searchServiceClient.GetSearchEndpointsAsync(regionInformation, token));
}
var maxCommit = DateTimeOffset.MinValue;
foreach (Instance instance in instances)
{
try
{
var commitDateTime = await _searchServiceClient.GetCommitDateTimeAsync(instance, token);
maxCommit = commitDateTime > maxCommit ? commitDateTime : maxCommit;
}
catch (Exception e)
{
Logger.LogError("An exception was encountered so no HTTP response was returned. {Exception}", e);
}
}
if (maxCommit == DateTimeOffset.MinValue)
{
Logger.LogError("Failed to retrieve a proper starting commit. Abandoning the current run.");
return;
}
var catalogLeafProcessor = new PackageLagCatalogLeafProcessor(instances, _searchServiceClient, _telemetryService, LoggerFactory.CreateLogger <PackageLagCatalogLeafProcessor>());
if (_configuration.RetryLimit > 0)
{
catalogLeafProcessor.RetryLimit = _configuration.RetryLimit;
}
if (_configuration.WaitBetweenRetrySeconds > 0)
{
catalogLeafProcessor.WaitBetweenPolls = TimeSpan.FromSeconds(_configuration.WaitBetweenRetrySeconds);
}
var settings = new CatalogProcessorSettings
{
ServiceIndexUrl = _configuration.ServiceIndexUrl,
DefaultMinCommitTimestamp = maxCommit,
ExcludeRedundantLeaves = false
};
var start = new FileCursor("cursor.json", LoggerFactory.CreateLogger <FileCursor>());
await start.SetAsync(maxCommit.AddTicks(1));
var catalogProcessor = new CatalogProcessor(start, _catalogClient, catalogLeafProcessor, settings, LoggerFactory.CreateLogger <CatalogProcessor>());
bool success;
int retryCount = 0;
do
{
success = await catalogProcessor.ProcessAsync();
if (!success || !await catalogLeafProcessor.WaitForProcessing())
{
retryCount++;
Logger.LogError("Processing the catalog leafs failed. Retry Count {CatalogProcessRetryCount}", retryCount);
}
}while (!success && retryCount < MAX_CATALOG_RETRY_COUNT);
return;
}
catch (Exception e)
{
Logger.LogError("Exception Occured. {Exception}", e);
return;
}
}
