private void DownloadMedia(TumblrTageedSearchApi post) { try { foreach (var data in post.Response.Posts.Data) { if (!CheckIfWithinTimespan(data.Timestamp)) { continue; } foreach (var content in data.Content) { DownloadMedia(content, data.Id, data.Timestamp, data.Tags); } AddToJsonQueue(new TumblrCrawlerData <Datum>(Path.ChangeExtension(data.Id, ".json"), data)); } } catch (TimeoutException timeoutException) { HandleTimeoutException(timeoutException, Resources.Crawling); } catch { } }
private async Task CrawlPageAsync() { try { string document = await GetTaggedSearchPageAsync(); string json = extractJsonFromSearch.Match(document).Groups[1].Value; TagSearch result = ConvertJsonToClass <TagSearch>(json); string nextUrl = result.ApiUrl + result.Tagged.NextLink.Href; string bearerToken = result.ApiFetchStore.APITOKEN; DownloadMedia(result); while (true) { if (CheckIfShouldStop()) { return; } CheckIfShouldPause(); document = await GetRequestAsync(nextUrl, bearerToken); TumblrTageedSearchApi apiresult = ConvertJsonToClass <TumblrTageedSearchApi>(document); nextUrl = result.ApiUrl + apiresult.Response.Posts.Links.Next.Href; DownloadMedia(apiresult); } } catch (TimeoutException timeoutException) { HandleTimeoutException(timeoutException, Resources.Crawling); } catch { } finally { semaphoreSlim.Release(); } }