private async Task ProcessPage(ShowIndexDto showIndexPage, CancellationToken cancellationToken) { foreach (var showId in showIndexPage.ShowIds) { if (cancellationToken.IsCancellationRequested) { return; } ShowDto show = await SafeExecute(() => dataProvider.GetShowCastAsync(showId)); if (show != null) { try { await AddShow(show); } catch (Exception e) { logger.LogError(e.ToString()); } } else { logger.LogWarning($"could not load show {showId}"); } } }
public async Task PerformScraping(CancellationToken cancellationToken) { bool hasMoreData; int page = 0; do { logger.LogInformation($"Fetching page {page}"); ShowIndexDto showIndexPage = await SafeExecute(() => dataProvider.GetShowIndexAsync(page)); if (showIndexPage != null && showIndexPage.HasData) { await ProcessPage(showIndexPage, cancellationToken); } hasMoreData = showIndexPage == null || showIndexPage.HasData; ++page; } while (hasMoreData); }