// scrape max 20 items from a random range of Show IDs
public async IAsyncEnumerable <TvMazeShowModel> ScrapeSome([EnumeratorCancellation] CancellationToken cancellationToken)
{
var maxNumberOfItemsToGet = 20;
var guessedNumberOfTotalItems = 1000;
int rangeStartIndex = new Random().Next(1, guessedNumberOfTotalItems);
var indexListToScrape = Enumerable.Range(rangeStartIndex, maxNumberOfItemsToGet);
foreach (var index in indexListToScrape)
{
TvMazeShowModel tvShowModel = await Scrape(index, cancellationToken);
yield return(tvShowModel);
}
}
private async Task <ShowModel> GetShowModelByTvMazeShowIdAsync(int tvMazeShowId)
{
TvMazeShowModel tvMazeShow = await _tvMazeApiService.GetTvMazeShowAsync(tvMazeShowId);
if (tvMazeShow != null)
{
tvMazeShow.Casts = await _tvMazeApiService.GetTvMazeCastsAsync(tvMazeShowId);
}
if (tvMazeShow?.Casts == null)
{
_logger.LogWarning(_strings[StringsEnum.INVALID_TVMAZE_DATA_RECEIVED]);
return(null);
}
return(_mapper.Map <ShowModel>(tvMazeShow));
}
private async Task <TvMazeShowModel> Scrape(int index, CancellationToken cancellationToken)
{
string scrapingUrl = BuildNextResourceUrl(index);
try
{
HttpResponseMessage response = await _httpClient.GetAsync(scrapingUrl, cancellationToken);
response.EnsureSuccessStatusCode();
string responseContent = await response.Content.ReadAsStringAsync();
TvMazeShowModel tvShowModel = GetModel(responseContent);
return(tvShowModel);
}
catch (Exception ex)
{
// ignore for now
_logger.LogError(ex, ex.Message);
return(null);
}
}
