public void Timeout()
{
HttpClientDownloader downloader = new HttpClientDownloader();
DefaultSpider spider = new DefaultSpider("abcd", new Site {
Timeout = 5000
});
downloader.Download(new Request("http://www.163.com", 0, null), spider);
try
{
downloader.Download(new Request("http://localhost/abcasdfasdfasdfas", 0, null), spider);
}
catch (Exception e)
{
bool r = e.Message == "Response status code does not indicate success: 503 (One or more errors occurred. (An error occurred while sending the request.))." || e.Message == "响应状态代码不指示成功: 503 (发生一个或多个错误。)。";
Assert.True(r);
}
Stopwatch watch = new Stopwatch();
watch.Start();
try
{
downloader.Download(new Request("http://google.com/", 0, null), spider);
}
catch (Exception e)
{
bool r = e.Message == "Response status code does not indicate success: 408 (Request Timeout)." || e.Message == "响应状态代码不指示成功: 408 (Request Timeout)。";
Assert.True(r);
}
watch.Stop();
Assert.True(watch.ElapsedMilliseconds > 5000);
Assert.True(watch.ElapsedMilliseconds < 6000);
}
public void Timeout()
{
HttpClientDownloader downloader = new HttpClientDownloader();
DefaultSpider spider = new DefaultSpider("abcd", new Site {
Timeout = 5000
});
downloader.Download(new Request("http://www.163.com", null), spider);
try
{
downloader.Download(new Request("http://localhost/abcasdfasdfasdfas", null), spider);
throw new Exception("Test Failed");
}
catch (SpiderException e)
{
Assert.IsNotNull(e);
}
Stopwatch watch = new Stopwatch();
watch.Start();
try
{
downloader.Download(new Request("http://google.com/", null), spider);
}
catch (SpiderException e)
{
Assert.IsNotNull(e);
}
watch.Stop();
Assert.IsTrue(watch.ElapsedMilliseconds > 5000);
Assert.IsTrue(watch.ElapsedMilliseconds < 6000);
}
public static string GetHtml(string name = "和壹（广州）品牌管理有限公司")
{
var _url = $"{"https:"}//www.qichacha.com/search?key={name}";
var _content = HttpClient.Download(new Request(_url)).Content.ToString();
var _content1 = HttpClient.Download(new Request(_url)).Content.ToString();
var _content2 = HttpClient.Download(new Request(_url)).Content.ToString();
var _content3 = HttpClient.Download(new Request(_url)).Content.ToString();
return(_content);
}
private static void MyTest()
{
Site site = new Site();
site.DownloadFiles = true;
site.Headers = new System.Collections.Generic.Dictionary <string, string>
{
{ "Cache-Control", "no-cache" },
{ "Pragma", "no-cache" },
{ "Accept", "image/webp,image/apng,image/*,*/*;q=0.8" },
{ "Accept-Encoding", "gzip, deflate, br" },
{ "Accept-Language", "zh-CN,zh;q=0.8" }
};
var spider = new DefaultSpider("test", site);
var request = new Request("https://cdn.dongmanmanhua.cn/15083718553482243295.jpg?x-oss-process=image/quality,q_90");
request.Referer = "https://www.dongmanmanhua.cn/fantasy/the-god-of-high-school/%E7%AC%AC6%E9%83%A8-re%E4%B8%8E%E7%A5%9E%E7%9A%84%E8%BE%83%E9%87%8F-%E7%AC%AC326%E8%AF%9D/viewer?title_no=224&episode_no=329";
var downloader = new HttpClientDownloader();
int downloadCount = 0;
var filePath = Path.Combine(AppContext.BaseDirectory, @"download\test\15083718553482243295.jpg");
for (int i = 0; i < 10; ++i)
{
downloader.Download(request, spider);
if (File.Exists(filePath))
{
File.Delete(filePath);
downloadCount++;
}
}
Console.WriteLine($"Download picture success: {downloadCount}");
}
public void GetTargetUrlWhenRedirect()
{
var downloader = new HttpClientDownloader();
var page = downloader.Download(new Request("http://item.jd.com/1231222221111123.html", null));
Assert.DoesNotContain("1231222221111123", page.TargetUrl);
Assert.True(page.TargetUrl.Contains("www.jd.com/") || page.TargetUrl.Contains("global.jd.com"));
}
public void Ports()
{
HttpClientDownloader downloader = new HttpClientDownloader();
for (int i = 0; i < 100; i++)
{
var a = downloader.Download(new Request("http://www.163.com", null));
}
}
public void GetTargetUrlWhenRedirect()
{
Site site = new Site
{
};
HttpClientDownloader downloader = new HttpClientDownloader();
var page = downloader.Download(new Request("http://item.jd.com/1231222221111123.html", null), new DefaultSpider("test", site));
Assert.EndsWith("www.jd.com/?d", page.TargetUrl);
}
public void Download()
{
HttpClientDownloader downloader = new HttpClientDownloader();
var response = downloader.Download(new Request("http://www.163.com")
{
Headers = new System.Collections.Generic.Dictionary <string, object>
{
{ "Cookies", "a=b" }
}
});
}
public void Ports()
{
HttpClientDownloader downloader = new HttpClientDownloader();
DefaultSpider spider = new DefaultSpider("abcd", new Site {
});
for (int i = 0; i < 100; i++)
{
var a = downloader.Download(new Request("http://www.163.com", null), spider).Result;
}
}
public void Ports()
{
HttpClientDownloader downloader = new HttpClientDownloader();
DefaultSpider spider = new DefaultSpider("abcd", new Site {
Timeout = 5000
});
for (int i = 0; i < 100; i++)
{
downloader.Download(new Request("http://www.163.com", 0, null), spider);
}
}
public void SetContentType()
{
Site site1 = new Site
{
Headers = new Dictionary <string, string>()
{
{ "Content-Type", "abcd" }
}
};
Site site2 = new Site
{
Headers = new Dictionary <string, string>()
{
{ "ContentType", "abcd" }
}
};
var downloader = new HttpClientDownloader();
var a = downloader.Download(new Request("http://163.com", null), new DefaultSpider("test", site1));
a = downloader.Download(new Request("http://163.com", null), new DefaultSpider("test", site2));
}
public void DetectDownloadContent()
{
HttpClientDownloader downloader = new HttpClientDownloader();
var a = downloader.Download(new Request("http://www.163.com", null));
Assert.Equal(ContentType.Html, a.ContentType);
HttpClientDownloader2 downloader2 = new HttpClientDownloader2();
a = downloader2.Download(new Request("http://www.163.com", null));
Assert.Equal(ContentType.Json, a.ContentType);
}
public void ParallelDownloader()
{
var downloader = new HttpClientDownloader();
Parallel.For(0, 3, new ParallelOptions
{
MaxDegreeOfParallelism = 3
}, i =>
{
var d = downloader.Clone();
downloader.Download(new Request("http://www.163.com"));
});
}
public void GetTargetUrlWhenRedirect()
{
Site site = new Site
{
Headers = new Dictionary <string, string>
{
{ "User-Agent", "Chrome" }
}
};
var downloader = new HttpClientDownloader();
var page = downloader.Download(new Request("http://item.jd.com/1231222221111123.html", null), new DefaultSpider("test", site));
Assert.True(page.TargetUrl.Contains("www.jd.com/2017?t=") || page.TargetUrl.Contains("global.jd.com"));
}
public void GetTargetUrlWhenRedirect()
{
Site site = new Site
{
Headers = new Dictionary <string, string>
{
{ "User-Agent", "Chrome" }
}
};
var downloader = new HttpClientDownloader();
var page = downloader.Download(new Request("http://item.jd.com/1231222221111123.html", null), new DefaultSpider("test", site));
Assert.EndsWith("www.jd.com/?d", page.TargetUrl);
}
public void Timeout()
{
HttpClientDownloader downloader = new HttpClientDownloader();
DefaultSpider spider = new DefaultSpider("abcd", new Site {
Timeout = 5000
});
downloader.Download(new Request("http://www.163.com", 0, null), spider);
Stopwatch watch = new Stopwatch();
watch.Start();
try
{
downloader.Download(new Request("http://www.xbxbxbxbxbxbxbxbxbxbxbxbxbxbxbxb.com", 0, null), spider);
}
catch (Exception e)
{
Assert.Equal("Response status code does not indicate success: 408 (Request Timeout).", e.Message);
}
watch.Stop();
Assert.True(watch.ElapsedMilliseconds > 5000);
Assert.True(watch.ElapsedMilliseconds < 6000);
}
private Response SendHeartbeartRequest()
{
var heartbeat = new NodeHeartbeatInput
{
Cpu = (int)EnvironmentUtil.GetCpuLoad(),
CpuCount = Environment.ProcessorCount,
FreeMemory = EnvironmentUtil.GetFreeMemory(),
Group = _options.Group,
Ip = EnvironmentUtil.IpAddress,
NodeId = NodeId.Id,
Os = EnvironmentUtil.OSDescription,
Runnings = _runnings.ToArray(),
TotalMemory = EnvironmentUtil.TotalMemory
};
Log.Logger.Information(heartbeat.ToString());
var response = _downloader.Download(new Request(_heartbeatUrl)
{
Method = HttpMethod.Post, Site = _brokerSite, Content = JsonConvert.SerializeObject(heartbeat)
});
return(response);
}
public void DetectDownloadContent()
{
HttpClientDownloader downloader = new HttpClientDownloader();
DefaultSpider spider = new DefaultSpider("abcd", new Site {
});
downloader.Download(new Request("http://www.163.com", null), spider);
Assert.Equal(Core.Infrastructure.ContentType.Html, spider.Site.ContentType);
HttpClientDownloader2 downloader2 = new HttpClientDownloader2();
DefaultSpider spider2 = new DefaultSpider("abcd", new Site {
});
downloader2.Download(new Request("http://www.163.com", null), spider2);
Assert.Equal(Core.Infrastructure.ContentType.Json, spider2.Site.ContentType);
}
public void TestStartAndStop()
{
HttpClientDownloader downloader = new HttpClientDownloader();
Core.Spider spider = Core.Spider.Create(new Site()
{
EncodingName = "UTF-8"
}, new SimplePageProcessor("http://www.oschina.net/", "http://www.oschina.net/*")).AddPipeline(new TestPipeline()).SetThreadNum(1);
Page p = downloader.Download(new Request("http://www.baidu.com/", 2, new Dictionary <string, dynamic>()), spider);
Console.WriteLine(p.Content);
spider.Start();
Thread.Sleep(10000);
spider.Stop();
Thread.Sleep(10000);
spider.Start();
Thread.Sleep(10000);
}
public async Task Download()
{
var client = new HttpClientDownloader();
using (var stream = new MemoryStream())
{
var obs = client.Download(new Uri(@"https://google.com/favicon.ico"), stream);
using (var d = obs.Connect())
{
var progress = await obs;
if (progress.Size.Bits != 0)
{
progress.Downloaded.Bits.Should().Be(progress.Size.Bits);
progress.Remaining.Bits.Should().Be(0);
}
progress.State.Should().Be(DownloadProgress.TransferState.Finished);
}
}
}
private List <string> GetHtmlContent(string id)
{
List <string> list = new List <string>();
HttpClientDownloader downloader = new HttpClientDownloader();
/*
* http://lizhi.yjbys.com/mingyan/373438.html
* http://lizhi.yjbys.com/mingyan/373443.html
* http://lizhi.yjbys.com/mingyan/221755.html
*/
string url = "http://lizhi.yjbys.com/mingyan/" + id + ".html";
Java2Dotnet.Spider.Core.Spider spider = Java2Dotnet.Spider.Core.Spider.Create(new Site()
{
EncodingName = "UTF-8"
},
new SimplePageProcessor(url, url + "/* ")).AddPipeline(new TestPipeline()).SetThreadNum(1);
Page p = downloader.Download(new Request(url, 2, new Dictionary <string, dynamic>()), spider);
//Console.WriteLine(p.Content);
var s = Selectors.Css("div p").SelectList(p.Content);
for (int i = 0; i < s.Count; i++)
{
Match m = Regex.Match(s[i].OuterHtml, @"<p>([\s\S]*)</p>", RegexOptions.IgnoreCase);
if (m.Success)
{
string result = m.Result("$1").Replace("—", "—").Replace(" ", "");
if (result != "")
{
if (!result.Contains("<"))
{
list.Add(result);
}
}
}
}
return(list);
}
/// <summary>
///
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public Page Get(string url)
{
return(httpClientDownloader.Download(new Request(url), site.ToTask()));
}
