private async Task ProcessPage(ShowIndexDto showIndexPage, CancellationToken cancellationToken)
{
foreach (var showId in showIndexPage.ShowIds)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
ShowDto show = await SafeExecute(() => dataProvider.GetShowCastAsync(showId));
if (show != null)
{
try
{
await AddShow(show);
}
catch (Exception e)
{
logger.LogError(e.ToString());
}
}
else
{
logger.LogWarning($"could not load show {showId}");
}
}
}
public async Task PerformScraping(CancellationToken cancellationToken)
{
bool hasMoreData;
int page = 0;
do
{
logger.LogInformation($"Fetching page {page}");
ShowIndexDto showIndexPage = await SafeExecute(() => dataProvider.GetShowIndexAsync(page));
if (showIndexPage != null && showIndexPage.HasData)
{
await ProcessPage(showIndexPage, cancellationToken);
}
hasMoreData = showIndexPage == null || showIndexPage.HasData;
++page;
} while (hasMoreData);
}