public async Task <ScrapeResult> ScrapeAsync(IScrapeSession session, CancellationToken cancellationToken = default)
{
var result = new ScrapeResult();
var options = _options.Value;
_logger.LogInformation("Scraping yts movies from {fromDate}", session.ScrapeFrom);
for (var page = 1; page < int.MaxValue; page++)
{
_logger.LogInformation("scraping page {page}", page);
var request = new YtsListMoviesRequest
{
Page = page,
Limit = 50,
OrderBy = "desc",
SortBy = "date_added"
};
var response = await _client.ListMoviesAsync(request, cancellationToken);
if (response.Movies is null)
{
break;
}
var movies = session.ScrapeFrom.HasValue
? response.Movies
.Where(x => x.DateUploaded > session.ScrapeFrom.Value)
.ToList()
: response.Movies;
if (!movies.Any())
{
break;
}
_logger.LogInformation("retrieved {movieCount} movies", movies.Count);
var requests = _mapper.Map <ICollection <CreateMovieRequest> >(movies);
await session.CreateMoviesAsync(requests);
_logger.LogInformation("added {movieCount} movies", requests.Count);
result.MovieCount += requests.Count;
result.TorrentCount += requests.Sum(x => x.Torrents?.Count ?? 0);
if (movies.Count < request.Limit)
{
break;
}
if (options.RemoteScrapeDelay > TimeSpan.Zero)
{
await Task.Delay(options.RemoteScrapeDelay, cancellationToken);
}
}
_logger.LogInformation("done");
return(result);
}
