public void Timeout() { HttpClientDownloader downloader = new HttpClientDownloader(); DefaultSpider spider = new DefaultSpider("abcd", new Site { Timeout = 5000 }); downloader.Download(new Request("http://www.163.com", 0, null), spider); try { downloader.Download(new Request("http://localhost/abcasdfasdfasdfas", 0, null), spider); } catch (Exception e) { bool r = e.Message == "Response status code does not indicate success: 503 (One or more errors occurred. (An error occurred while sending the request.))." || e.Message == "响应状态代码不指示成功: 503 (发生一个或多个错误。)。"; Assert.True(r); } Stopwatch watch = new Stopwatch(); watch.Start(); try { downloader.Download(new Request("http://google.com/", 0, null), spider); } catch (Exception e) { bool r = e.Message == "Response status code does not indicate success: 408 (Request Timeout)." || e.Message == "响应状态代码不指示成功: 408 (Request Timeout)。"; Assert.True(r); } watch.Stop(); Assert.True(watch.ElapsedMilliseconds > 5000); Assert.True(watch.ElapsedMilliseconds < 6000); }
public void Timeout() { HttpClientDownloader downloader = new HttpClientDownloader(); DefaultSpider spider = new DefaultSpider("abcd", new Site { Timeout = 5000 }); downloader.Download(new Request("http://www.163.com", null), spider); try { downloader.Download(new Request("http://localhost/abcasdfasdfasdfas", null), spider); throw new Exception("Test Failed"); } catch (SpiderException e) { Assert.IsNotNull(e); } Stopwatch watch = new Stopwatch(); watch.Start(); try { downloader.Download(new Request("http://google.com/", null), spider); } catch (SpiderException e) { Assert.IsNotNull(e); } watch.Stop(); Assert.IsTrue(watch.ElapsedMilliseconds > 5000); Assert.IsTrue(watch.ElapsedMilliseconds < 6000); }
public static string GetHtml(string name = "和壹(广州)品牌管理有限公司") { var _url = $"{"https:"}//www.qichacha.com/search?key={name}"; var _content = HttpClient.Download(new Request(_url)).Content.ToString(); var _content1 = HttpClient.Download(new Request(_url)).Content.ToString(); var _content2 = HttpClient.Download(new Request(_url)).Content.ToString(); var _content3 = HttpClient.Download(new Request(_url)).Content.ToString(); return(_content); }
private static void MyTest() { Site site = new Site(); site.DownloadFiles = true; site.Headers = new System.Collections.Generic.Dictionary <string, string> { { "Cache-Control", "no-cache" }, { "Pragma", "no-cache" }, { "Accept", "image/webp,image/apng,image/*,*/*;q=0.8" }, { "Accept-Encoding", "gzip, deflate, br" }, { "Accept-Language", "zh-CN,zh;q=0.8" } }; var spider = new DefaultSpider("test", site); var request = new Request("https://cdn.dongmanmanhua.cn/15083718553482243295.jpg?x-oss-process=image/quality,q_90"); request.Referer = "https://www.dongmanmanhua.cn/fantasy/the-god-of-high-school/%E7%AC%AC6%E9%83%A8-re%E4%B8%8E%E7%A5%9E%E7%9A%84%E8%BE%83%E9%87%8F-%E7%AC%AC326%E8%AF%9D/viewer?title_no=224&episode_no=329"; var downloader = new HttpClientDownloader(); int downloadCount = 0; var filePath = Path.Combine(AppContext.BaseDirectory, @"download\test\15083718553482243295.jpg"); for (int i = 0; i < 10; ++i) { downloader.Download(request, spider); if (File.Exists(filePath)) { File.Delete(filePath); downloadCount++; } } Console.WriteLine($"Download picture success: {downloadCount}"); }
public void GetTargetUrlWhenRedirect() { var downloader = new HttpClientDownloader(); var page = downloader.Download(new Request("http://item.jd.com/1231222221111123.html", null)); Assert.DoesNotContain("1231222221111123", page.TargetUrl); Assert.True(page.TargetUrl.Contains("www.jd.com/") || page.TargetUrl.Contains("global.jd.com")); }
public void Ports() { HttpClientDownloader downloader = new HttpClientDownloader(); for (int i = 0; i < 100; i++) { var a = downloader.Download(new Request("http://www.163.com", null)); } }
public void GetTargetUrlWhenRedirect() { Site site = new Site { }; HttpClientDownloader downloader = new HttpClientDownloader(); var page = downloader.Download(new Request("http://item.jd.com/1231222221111123.html", null), new DefaultSpider("test", site)); Assert.EndsWith("www.jd.com/?d", page.TargetUrl); }
public void Download() { HttpClientDownloader downloader = new HttpClientDownloader(); var response = downloader.Download(new Request("http://www.163.com") { Headers = new System.Collections.Generic.Dictionary <string, object> { { "Cookies", "a=b" } } }); }
public void Ports() { HttpClientDownloader downloader = new HttpClientDownloader(); DefaultSpider spider = new DefaultSpider("abcd", new Site { }); for (int i = 0; i < 100; i++) { var a = downloader.Download(new Request("http://www.163.com", null), spider).Result; } }
public void Ports() { HttpClientDownloader downloader = new HttpClientDownloader(); DefaultSpider spider = new DefaultSpider("abcd", new Site { Timeout = 5000 }); for (int i = 0; i < 100; i++) { downloader.Download(new Request("http://www.163.com", 0, null), spider); } }
public void SetContentType() { Site site1 = new Site { Headers = new Dictionary <string, string>() { { "Content-Type", "abcd" } } }; Site site2 = new Site { Headers = new Dictionary <string, string>() { { "ContentType", "abcd" } } }; var downloader = new HttpClientDownloader(); var a = downloader.Download(new Request("http://163.com", null), new DefaultSpider("test", site1)); a = downloader.Download(new Request("http://163.com", null), new DefaultSpider("test", site2)); }
public void DetectDownloadContent() { HttpClientDownloader downloader = new HttpClientDownloader(); var a = downloader.Download(new Request("http://www.163.com", null)); Assert.Equal(ContentType.Html, a.ContentType); HttpClientDownloader2 downloader2 = new HttpClientDownloader2(); a = downloader2.Download(new Request("http://www.163.com", null)); Assert.Equal(ContentType.Json, a.ContentType); }
public void ParallelDownloader() { var downloader = new HttpClientDownloader(); Parallel.For(0, 3, new ParallelOptions { MaxDegreeOfParallelism = 3 }, i => { var d = downloader.Clone(); downloader.Download(new Request("http://www.163.com")); }); }
public void GetTargetUrlWhenRedirect() { Site site = new Site { Headers = new Dictionary <string, string> { { "User-Agent", "Chrome" } } }; var downloader = new HttpClientDownloader(); var page = downloader.Download(new Request("http://item.jd.com/1231222221111123.html", null), new DefaultSpider("test", site)); Assert.True(page.TargetUrl.Contains("www.jd.com/2017?t=") || page.TargetUrl.Contains("global.jd.com")); }
public void GetTargetUrlWhenRedirect() { Site site = new Site { Headers = new Dictionary <string, string> { { "User-Agent", "Chrome" } } }; var downloader = new HttpClientDownloader(); var page = downloader.Download(new Request("http://item.jd.com/1231222221111123.html", null), new DefaultSpider("test", site)); Assert.EndsWith("www.jd.com/?d", page.TargetUrl); }
public void Timeout() { HttpClientDownloader downloader = new HttpClientDownloader(); DefaultSpider spider = new DefaultSpider("abcd", new Site { Timeout = 5000 }); downloader.Download(new Request("http://www.163.com", 0, null), spider); Stopwatch watch = new Stopwatch(); watch.Start(); try { downloader.Download(new Request("http://www.xbxbxbxbxbxbxbxbxbxbxbxbxbxbxbxb.com", 0, null), spider); } catch (Exception e) { Assert.Equal("Response status code does not indicate success: 408 (Request Timeout).", e.Message); } watch.Stop(); Assert.True(watch.ElapsedMilliseconds > 5000); Assert.True(watch.ElapsedMilliseconds < 6000); }
private Response SendHeartbeartRequest() { var heartbeat = new NodeHeartbeatInput { Cpu = (int)EnvironmentUtil.GetCpuLoad(), CpuCount = Environment.ProcessorCount, FreeMemory = EnvironmentUtil.GetFreeMemory(), Group = _options.Group, Ip = EnvironmentUtil.IpAddress, NodeId = NodeId.Id, Os = EnvironmentUtil.OSDescription, Runnings = _runnings.ToArray(), TotalMemory = EnvironmentUtil.TotalMemory }; Log.Logger.Information(heartbeat.ToString()); var response = _downloader.Download(new Request(_heartbeatUrl) { Method = HttpMethod.Post, Site = _brokerSite, Content = JsonConvert.SerializeObject(heartbeat) }); return(response); }
public void DetectDownloadContent() { HttpClientDownloader downloader = new HttpClientDownloader(); DefaultSpider spider = new DefaultSpider("abcd", new Site { }); downloader.Download(new Request("http://www.163.com", null), spider); Assert.Equal(Core.Infrastructure.ContentType.Html, spider.Site.ContentType); HttpClientDownloader2 downloader2 = new HttpClientDownloader2(); DefaultSpider spider2 = new DefaultSpider("abcd", new Site { }); downloader2.Download(new Request("http://www.163.com", null), spider2); Assert.Equal(Core.Infrastructure.ContentType.Json, spider2.Site.ContentType); }
public void TestStartAndStop() { HttpClientDownloader downloader = new HttpClientDownloader(); Core.Spider spider = Core.Spider.Create(new Site() { EncodingName = "UTF-8" }, new SimplePageProcessor("http://www.oschina.net/", "http://www.oschina.net/*")).AddPipeline(new TestPipeline()).SetThreadNum(1); Page p = downloader.Download(new Request("http://www.baidu.com/", 2, new Dictionary <string, dynamic>()), spider); Console.WriteLine(p.Content); spider.Start(); Thread.Sleep(10000); spider.Stop(); Thread.Sleep(10000); spider.Start(); Thread.Sleep(10000); }
public async Task Download() { var client = new HttpClientDownloader(); using (var stream = new MemoryStream()) { var obs = client.Download(new Uri(@"https://google.com/favicon.ico"), stream); using (var d = obs.Connect()) { var progress = await obs; if (progress.Size.Bits != 0) { progress.Downloaded.Bits.Should().Be(progress.Size.Bits); progress.Remaining.Bits.Should().Be(0); } progress.State.Should().Be(DownloadProgress.TransferState.Finished); } } }
private List <string> GetHtmlContent(string id) { List <string> list = new List <string>(); HttpClientDownloader downloader = new HttpClientDownloader(); /* * http://lizhi.yjbys.com/mingyan/373438.html * http://lizhi.yjbys.com/mingyan/373443.html * http://lizhi.yjbys.com/mingyan/221755.html */ string url = "http://lizhi.yjbys.com/mingyan/" + id + ".html"; Java2Dotnet.Spider.Core.Spider spider = Java2Dotnet.Spider.Core.Spider.Create(new Site() { EncodingName = "UTF-8" }, new SimplePageProcessor(url, url + "/* ")).AddPipeline(new TestPipeline()).SetThreadNum(1); Page p = downloader.Download(new Request(url, 2, new Dictionary <string, dynamic>()), spider); //Console.WriteLine(p.Content); var s = Selectors.Css("div p").SelectList(p.Content); for (int i = 0; i < s.Count; i++) { Match m = Regex.Match(s[i].OuterHtml, @"<p>([\s\S]*)</p>", RegexOptions.IgnoreCase); if (m.Success) { string result = m.Result("$1").Replace("—", "—").Replace(" ", ""); if (result != "") { if (!result.Contains("<")) { list.Add(result); } } } } return(list); }
/// <summary> /// /// </summary> /// <param name="url"></param> /// <returns></returns> public Page Get(string url) { return(httpClientDownloader.Download(new Request(url), site.ToTask())); }