public void Download(DownloadServiceOption options)
{
var worker = new WebRequestWorker();
foreach (var urlAndPattern in options.UrlsAndPatterns)
{
var uri = urlAndPattern.Key;
var configFile = urlAndPattern.Value;
Console.WriteLine("Start processing " + uri);
var categoryResult = worker.DownloadResponse(new CrawlingOption(uri));
if (categoryResult.StatusCode == HttpStatusCode.OK)
{
var configs = SerializationHelper.DeserializeFrom <List <DataItem> >(configFile);
var pageLinkPattern = configs.Get(Constants.PagePattern).Value;
var fileLinkPattern = configs.Get(Constants.FilePattern).Value;
var categoryPages = GetArticleLinks(categoryResult.ReadAsText(), pageLinkPattern);
ProcessAllPages(categoryPages, fileLinkPattern, options.TargetFolder);
}
else
{
Console.WriteLine(categoryResult.StatusDescription);
}
}
}
public static IHtmlString GetRemoteContent(this HtmlHelper html, string fileUri)
{
if (string.IsNullOrWhiteSpace(fileUri))
{
return(MvcHtmlString.Empty);
}
var fileContent = new WebRequestWorker().DownloadResponse(new CrawlingOption(fileUri)).ReadAsText();
return(html.Raw(fileContent));
}
public WebRequestBuilder(IEnumerable <IContentSerializer> serializers, ContentType defaultContentType, Action <WebRequest> webrequestModifier, IEnumerable <Credential> credentials)
{
var worker = new WebRequestWorker(webrequestModifier, ClientUtils.CreateCredentials(credentials));
_tools = new WebRequestTools
{
DefaultContentType = defaultContentType == ContentType.Unknown ? ContentType.ApplicationJson : defaultContentType,
Serializers = serializers.ToList(),
RequestModifier = webrequestModifier,
Resenders = new List <IRequestResender>
{
new UnsupportedMediaTypeResender(serializers.ToList(), worker)
},
RequestWorker = worker
};
}
public static T DeserializeFrom <T>(string filePath) where T : class
{
if (filePath.Contains("://"))
{
var result = new WebRequestWorker().DownloadResponse(new CrawlingOption(filePath));
if (result.StatusCode == HttpStatusCode.OK)
{
return(result.ReadAsText().Deserialize <T>());
}
}
else
{
return(File.ReadAllText(filePath).Deserialize <T>());
}
return(default(T));
}
