public void Download(DownloadServiceOption options) { var worker = new WebRequestWorker(); foreach (var urlAndPattern in options.UrlsAndPatterns) { var uri = urlAndPattern.Key; var configFile = urlAndPattern.Value; Console.WriteLine("Start processing " + uri); var categoryResult = worker.DownloadResponse(new CrawlingOption(uri)); if (categoryResult.StatusCode == HttpStatusCode.OK) { var configs = SerializationHelper.DeserializeFrom <List <DataItem> >(configFile); var pageLinkPattern = configs.Get(Constants.PagePattern).Value; var fileLinkPattern = configs.Get(Constants.FilePattern).Value; var categoryPages = GetArticleLinks(categoryResult.ReadAsText(), pageLinkPattern); ProcessAllPages(categoryPages, fileLinkPattern, options.TargetFolder); } else { Console.WriteLine(categoryResult.StatusDescription); } } }
public static IHtmlString GetRemoteContent(this HtmlHelper html, string fileUri) { if (string.IsNullOrWhiteSpace(fileUri)) { return(MvcHtmlString.Empty); } var fileContent = new WebRequestWorker().DownloadResponse(new CrawlingOption(fileUri)).ReadAsText(); return(html.Raw(fileContent)); }
public WebRequestBuilder(IEnumerable <IContentSerializer> serializers, ContentType defaultContentType, Action <WebRequest> webrequestModifier, IEnumerable <Credential> credentials) { var worker = new WebRequestWorker(webrequestModifier, ClientUtils.CreateCredentials(credentials)); _tools = new WebRequestTools { DefaultContentType = defaultContentType == ContentType.Unknown ? ContentType.ApplicationJson : defaultContentType, Serializers = serializers.ToList(), RequestModifier = webrequestModifier, Resenders = new List <IRequestResender> { new UnsupportedMediaTypeResender(serializers.ToList(), worker) }, RequestWorker = worker }; }
public static T DeserializeFrom <T>(string filePath) where T : class { if (filePath.Contains("://")) { var result = new WebRequestWorker().DownloadResponse(new CrawlingOption(filePath)); if (result.StatusCode == HttpStatusCode.OK) { return(result.ReadAsText().Deserialize <T>()); } } else { return(File.ReadAllText(filePath).Deserialize <T>()); } return(default(T)); }