/// <summary>
/// 抓取页面字节流
/// </summary>
/// <param name="webClient">Web客户端</param>
/// <param name="onCrawlData">抓取回调事件</param>
/// <param name="request">URI请求信息</param>
private void crawl(webClient webClient, Action <byte[]> onCrawlData, ref request request)
{
this.request = request;
this.webClient = webClient;
this.onCrawlData = onCrawlData;
if (request.Uri != null)
{
try
{
webClient.Headers.Add(header.UserAgent, webClient.UserAgent);
webClient.Headers.Add(header.Referer, request.RefererUrl == null || request.RefererUrl.Length == 0 ? request.Uri.AbsoluteUri : request.RefererUrl);
if (request.Form == null)
{
if (request.UploadData == null)
{
downloadData();
}
else
{
upload();
}
}
else
{
postForm();
}
return;
}
catch (Exception error)
{
webClient.onError(error, ref request);
}
}
onCrawl(null);
}
///// <summary>
///// JS类型
///// </summary>
//private static readonly string jsonContentType = "application/json; charset=" + Encoding.ASCII.WebName;
/// <summary>
/// 启动负载均衡服务
/// </summary>
/// <returns>是否成功</returns>
public static bool LoadBalancing()
{
using (webClient webClient = new webClient())
{
webClient.KeepAlive = false;
return(fastCSharp.emit.jsonParser.Parse <fastCSharp.code.cSharp.tcpBase.parameterJsonToSerialize <int> >(webClient.CrawlHtml("http://" + webConfig.config.Default.Domain + "/ajax?n=loadBalancing.Start", Encoding.ASCII)).Return != 0);
}
}
/// <summary>
/// 抓取回调
/// </summary>
/// <param name="data">抓取数据</param>
private void onCrawl(byte[] data)
{
try
{
onCrawlData(data);
}
finally
{
webClient = null;
onCrawlData = null;
request.Clear();
typePool <dataCrawler> .PushNotNull(this);
}
}
/// <summary>
/// 异步抓取页面HTML代码
/// </summary>
/// <param name="webClient">Web客户端</param>
/// <param name="onCrawlData">抓取回调事件</param>
/// <param name="request">URI请求信息</param>
public static void Crawl(webClient webClient, Action <byte[]> onCrawlData, ref request request)
{
dataCrawler dataCrawler = typePool <dataCrawler> .Pop();
if (dataCrawler == null)
{
try
{
dataCrawler = new dataCrawler();
}
catch (Exception error)
{
log.Default.Add(error, null, false);
onCrawlData(null);
return;
}
}
dataCrawler.crawl(webClient, onCrawlData, ref request);
}
