public async Task ProcessAsync(ICrawler crawler, PropertyBag propertyBag)
{
if (propertyBag.StatusCode != HttpStatusCode.OK)
{
return;
}
var extension = MapContentTypeToExtension(propertyBag.ContentType);
if (extension.IsNullOrEmpty())
{
return;
}
propertyBag.Title = propertyBag.Step.Uri.PathAndQuery;
using (var temp = new TempFile())
{
temp.FileName += "." + extension;
using (var fs = new FileStream(temp.FileName, FileMode.Create, FileAccess.Write, FileShare.Read, 0x1000))
using (var input = propertyBag.GetResponse())
{
input.CopyTo(fs);
}
using (var filterReader = new FilterReader(temp.FileName))
{
var content = await filterReader.ReadToEndAsync();
propertyBag.Text = content.Trim();
}
}
}