private void DownloadMedia(TumblrTageedSearchApi post)
{
try
{
foreach (var data in post.Response.Posts.Data)
{
if (!CheckIfWithinTimespan(data.Timestamp))
{
continue;
}
foreach (var content in data.Content)
{
DownloadMedia(content, data.Id, data.Timestamp, data.Tags);
}
AddToJsonQueue(new TumblrCrawlerData <Datum>(Path.ChangeExtension(data.Id, ".json"), data));
}
}
catch (TimeoutException timeoutException)
{
HandleTimeoutException(timeoutException, Resources.Crawling);
}
catch
{
}
}
private async Task CrawlPageAsync()
{
try
{
string document = await GetTaggedSearchPageAsync();
string json = extractJsonFromSearch.Match(document).Groups[1].Value;
TagSearch result = ConvertJsonToClass <TagSearch>(json);
string nextUrl = result.ApiUrl + result.Tagged.NextLink.Href;
string bearerToken = result.ApiFetchStore.APITOKEN;
DownloadMedia(result);
while (true)
{
if (CheckIfShouldStop())
{
return;
}
CheckIfShouldPause();
document = await GetRequestAsync(nextUrl, bearerToken);
TumblrTageedSearchApi apiresult = ConvertJsonToClass <TumblrTageedSearchApi>(document);
nextUrl = result.ApiUrl + apiresult.Response.Posts.Links.Next.Href;
DownloadMedia(apiresult);
}
}
catch (TimeoutException timeoutException)
{
HandleTimeoutException(timeoutException, Resources.Crawling);
}
catch
{
}
finally
{
semaphoreSlim.Release();
}
}