private async Task <int> ScrapeImagesAsync(IScrapeSession session, CancellationToken cancellationToken = default)
{
_logger.LogInformation("scraping images");
session.CreateMovieImageDirectory();
const int limit = 50;
var result = await _movieService.GetMoviesWithMissingImagesAsync(limit, 0, cancellationToken);
_logger.LogInformation("starting image scrape {count}", result.Count);
if (result.Count == 0)
{
return(0);
}
var options = _options.Value;
var scraped = 0;
var skip = 0;
while (result.Any())
{
_logger.LogInformation("scraping images for movie ids {from} to {to}", result.First().Id, result.Last().Id);
foreach (var sources in result)
{
if (await session.AssertMovieImageAsync(sources.ImdbCode, cancellationToken))
{
_logger.LogInformation("successfully updated image from local filesystem");
continue;
}
// prefer local image
if (await ScrapeImageAsync(session, sources.ImdbCode, sources.LocalSource, cancellationToken))
{
_logger.LogInformation("successfully updated image from local movie source {source}", sources.LocalSource);
scraped++;
continue;
}
var success = false;
foreach (var source in sources.RemoteSources)
{
success = await ScrapeImageAsync(session, sources.ImdbCode, source, cancellationToken);
if (success)
{
_logger.LogInformation("successfully updated image from remote movie source {source}", source.Source);
scraped++;
}
else
{
_logger.LogWarning("failed to scrape {image} from {source}", source.Value, source.Source);
}
if (options.RemoteScrapeDelay > TimeSpan.Zero)
{
await Task.Delay(options.RemoteScrapeDelay, cancellationToken);
}
if (success)
{
break;
}
}
if (!success)
{
// skip this image next time.
skip++;
}
}
result = await _movieService.GetMoviesWithMissingImagesAsync(limit, skip, cancellationToken);
}
_logger.LogInformation("done scraping images");
return(scraped);
}
