public static async Task<IPicture> ScrapeAsync(IScraper scraper)
{
using (var client = new HttpClient())
{
// It looks like dilbert.com mucks with the charset ... intentionally?
//var html = await client.GetStringAsync(scraper.Url);
HttpResponseMessage response = await client.GetAsync(scraper.Url);
response.EnsureSuccessStatusCode();
string charset = response.Content.Headers.ContentType.CharSet;
if (charset.EndsWith("lias"))
{
charset = charset.Replace("lias", String.Empty);
response.Content.Headers.ContentType.CharSet = charset;
}
string html = await response.Content.ReadAsStringAsync();
Uri imageUrl = await scraper.GetImageUrlAsync(html);
// be sure the URL is really an image
//Image image = await GetImageAsync(imageUrl);
//byte[] imageBytes = await client.GetByteArrayAsync(imageUrl);
IImage image = await GetImageAsync(imageUrl);
return new Picture(scraper, imageUrl, image);
}
}
