public async Task DownloadAsync()
{
_logger.LogDebug("Configuration:");
using (_logger.Indent())
{
_logger.LogDebug("User-Agent: {UserAgent}", _userAgent);
_logger.LogDebug("Service index: {ServiceIndexUrl}", _config.ServiceIndexUrl);
_logger.LogDebug("Data directory: {DataDirectory}", _config.DataDirectory);
_logger.LogDebug("Depth: {Depth}", _config.Depth);
_logger.LogDebug("JSON formatting: {JsonFormatting}", _config.JsonFormatting);
_logger.LogDebug("Max pages: {MaxPages}", _config.MaxPages);
_logger.LogDebug("Max commits: {MaxCommits}", _config.MaxCommits);
_logger.LogDebug("Save to disk: {SaveToDisk}", _config.SaveToDisk);
_logger.LogDebug("Format paths: {FormatPaths}", _config.FormatPaths);
_logger.LogDebug("Parallel downloads: {ParallelDownloads}", _config.ParallelDownloads);
}
_logger.LogDebug("Starting..." + Environment.NewLine);
if (_config.MaxCommits.HasValue && _config.Depth < DownloadDepth.CatalogPage)
{
throw new InvalidOperationException($"The download depth must be at least {DownloadDepth.CatalogPage} when setting a maximum number of commits.");
}
if (_config.MaxPages.HasValue && _config.Depth < DownloadDepth.CatalogIndex)
{
throw new InvalidOperationException($"The download depth must be at least {DownloadDepth.CatalogIndex} when setting a maximum number of pages.");
}
_logger.LogInformation($"Downloading service index: {_config.ServiceIndexUrl}");
var serviceIndex = await DownloadAndParseAsync <ServiceIndex>(_config.ServiceIndexUrl);
if (_config.Depth == DownloadDepth.ServiceIndex)
{
return;
}
const string catalogResourceType = "Catalog/3.0.0";
var catalogResource = serviceIndex.Value.Resources.SingleOrDefault(x => x.Type == catalogResourceType);
if (catalogResource == null)
{
throw new InvalidOperationException($"No {catalogResourceType} resource was found in '{_config.ServiceIndexUrl}'.");
}
await ProcessCatalogAsync(catalogResource.Url);
}
private async Task UpdateAsync(ReportName name, Func <string, IVisitor> getVisitor)
{
_logger.LogInformation("Updating report {Name}.", name);
using (_logger.Indent())
{
var cursorProvider = new CursorFactory(
cursorSuffix: $"report.{name}",
defaultCursorValue: _defaultCursorValue,
logger: _logger);
var csvPath = Path.Combine(_config.DataDirectory, "reports", $"{name}.csv");
var downloader = new Downloader(
_httpClient,
_config,
cursorProvider,
getVisitor(csvPath),
_logger);
await downloader.DownloadAsync();
}
}
