private IEnumerable<MainItem> GetCaoQunMainItems()
{
List<MainItem> res = new List<MainItem>();
HttpCore hc = new HttpCore();
hc.SetUrl(CaoQunUrl);
hc.CurrentHttpItem.Allowautoredirect = true;
string mainHtml = hc.GetHtml();
HtmlNodeCollection mainItems = mainHtml.SelectNodes("//*[@id='ajaxtable']/tbody[1]/tr");
if (mainItems == null)
{
return new List<MainItem>();
}
foreach (HtmlNode mainItem in mainItems)
{
var an = mainItem.SelectSingleNode(mainItem.XPath + "/td[2]/h3/a");
if (an == null)
{
continue;
}
string infoUrl = "http://5.yao.cl/" + an.Attributes["href"].Value;
string title = an.InnerText;
MainItem temp = new MainItem
{
HPLDetailItem = new HPLDetailItem(),
InfoUrl = infoUrl,
Title = title,
Url = CaoQunUrl
};
res.Add(temp);
}
return res;
}
private List <MainItem> GetCaoQunMainItems()
{
List <MainItem> res = new List <MainItem>();
HttpCore hc = new HttpCore();
hc.SetUrl(CaoQunUrl);
string mainHtml = hc.GetHtml();
//MSScriptControl.ScriptControlClass scc = new MSScriptControl.ScriptControlClass { Language = "javascript" };
//mainHtml = scc.Eval(mainHtml).ToString();
ThreadWebBrowser(CaoQunUrl);
while (kM == null)
{
Thread.Sleep(1000);
}
HtmlNodeCollection mainItems = mainHtml.SelectNodes("/html/body/div[8]/div[2]/div[@class='list-pianyuan-box']");
if (mainItems == null)
{
return(new List <MainItem>());
}
foreach (HtmlNode mainItem in mainItems)
{
var an = mainItem.SelectSingleNode("/div[1]/div[2]/div[1]/a");
string infoUrl = "http://www.haipilu.net/" + an.Attributes["href"].Value;
string title = an.InnerText;
MainItem temp = new MainItem
{
HPLDetailItem = new HPLDetailItem(),
InfoUrl = infoUrl,
Title = title,
Url = CaoQunUrl
};
res.Add(temp);
}
return(res);
}
public void ExecCrawler()
{
List <CaoQunMainItem> mainItems = GetCaoQunMainItems();
Parallel.ForEach(mainItems, caoQunMainItem =>
{
HttpCore hc1 = new HttpCore();
hc1.SetUrl(caoQunMainItem.InfoUrl);
string infoHtml = hc1.GetHtml();
HtmlNodeCollection imgNodes = infoHtml.SelectNodes("//*[@id='read_tpc']/img");
if (imgNodes == null)
{
return;
}
string remark = infoHtml.SelectNodes("//*[@id='read_tpc']")[0].InnerText;
caoQunMainItem.Remark = remark;
Parallel.ForEach(imgNodes, htmlNode =>
{
string imgUrl = htmlNode.Attributes["src"].Value;
HttpCore hc = new HttpCore {
CurrentHttpItem = { ResultType = ResultType.Byte }
};
hc.SetUrl(imgUrl);
HttpResult temp = hc.GetHttpResult();
//把得到的Byte转成图片
Image img = temp.ResultByte.ByteArrayToImage();
if (img == null)
{
return;
}
SaveImg("img2", caoQunMainItem, img, imgUrl);
});
//数据入库
StoreDB(caoQunMainItem);
});
}
public void ExecCrawler()
{
List<MainItem> mainItems = GetCaoQunMainItems();
Parallel.ForEach(mainItems, caoQunMainItem =>
{
HttpCore hc1 = new HttpCore();
hc1.SetUrl(caoQunMainItem.InfoUrl);
string infoHtml = hc1.GetHtml();
HtmlNodeCollection imgNodes = infoHtml.SelectNodes("/html/body/div[7]/div/div[4]/div[2]/div/img");
if (imgNodes == null)
{
return;
}
string remark = infoHtml.SelectNodes("/html/body/div[7]/div/div[4]/div[2]/div/img")[0].InnerText;
caoQunMainItem.Remark = remark;
Parallel.ForEach(imgNodes, htmlNode =>
{
string imgUrl = htmlNode.Attributes["src"].Value;
HttpCore hc = new HttpCore { CurrentHttpItem = { ResultType = ResultType.Byte } };
hc.SetUrl(imgUrl);
HttpResult temp = hc.GetHttpResult();
//把得到的Byte转成图片
Image img = temp.ResultByte.ByteArrayToImage();
if (img == null)
{
return;
}
SaveImg("img5", caoQunMainItem, img, imgUrl);
});
//数据入库
StoreDB(caoQunMainItem);
});
}
private IEnumerable <MainItem> GetCaoQunMainItems()
{
List <MainItem> res = new List <MainItem>();
HttpCore hc = new HttpCore();
hc.SetUrl(CaoQunUrl);
hc.CurrentHttpItem.Allowautoredirect = true;
string mainHtml = hc.GetHtml();
HtmlNodeCollection mainItems = mainHtml.SelectNodes("//*[@id='ajaxtable']/tbody[1]/tr");
if (mainItems == null)
{
return(new List <MainItem>());
}
foreach (HtmlNode mainItem in mainItems)
{
var an = mainItem.SelectSingleNode(mainItem.XPath + "/td[2]/h3/a");
if (an == null)
{
continue;
}
string infoUrl = "http://5.yao.cl/" + an.Attributes["href"].Value;
string title = an.InnerText;
MainItem temp = new MainItem
{
HPLDetailItem = new HPLDetailItem(),
InfoUrl = infoUrl,
Title = title,
Url = CaoQunUrl
};
res.Add(temp);
}
return(res);
}
private List<MainItem> GetCaoQunMainItems()
{
List<MainItem> res = new List<MainItem>();
HttpCore hc = new HttpCore();
hc.SetUrl(CaoQunUrl);
string mainHtml = hc.GetHtml();
//MSScriptControl.ScriptControlClass scc = new MSScriptControl.ScriptControlClass { Language = "javascript" };
//mainHtml = scc.Eval(mainHtml).ToString();
ThreadWebBrowser(CaoQunUrl);
while (kM == null)
{
Thread.Sleep(1000);
}
HtmlNodeCollection mainItems = mainHtml.SelectNodes("/html/body/div[8]/div[2]/div[@class='list-pianyuan-box']");
if (mainItems == null)
{
return new List<MainItem>();
}
foreach (HtmlNode mainItem in mainItems)
{
var an = mainItem.SelectSingleNode("/div[1]/div[2]/div[1]/a");
string infoUrl = "http://www.haipilu.net/" + an.Attributes["href"].Value;
string title = an.InnerText;
MainItem temp = new MainItem
{
HPLDetailItem = new HPLDetailItem(),
InfoUrl = infoUrl,
Title = title,
Url = CaoQunUrl
};
res.Add(temp);
}
return res;
}
public void Test()
{
Parallel.For(0, 100, i =>
{
DateTime dt = SysTimeRecord.Get(i.ToString());
SysTimeRecord.Set(i.ToString(), dt == DateTime.MinValue ? DateTime.Now.AddDays(-i) : dt.AddHours(1));
});
for (int i = 0; i < 100; i++)
{
DateTime dt = SysTimeRecord.Get(i.ToString());
SysTimeRecord.Set(i.ToString(), dt == DateTime.MinValue ? DateTime.Now.AddDays(i) : dt.AddHours(-1));
}
RedisTest.Test();
AsyncWorker<int> aw = new AsyncWorker<int>(3, Console.WriteLine);
for (int i = 0; i < 100; i++)
{
aw.AddItem(i);
}
aw.StopAndWait();
Thread.Sleep(10000000);
Parallel.Invoke(
() => ExecCrawler(1),
() => ExecCrawler(2),
() => ExecCrawler(3),
() => ExecCrawler(4),
() => ExecCrawler(5)
);
HttpCore hc = new HttpCore();
hc.SetUrl("http://ac168.info/bt/thread.php?fid=16&page={0}", 2);
var res = hc.GetHtml();
var tres = res.SelectNodes("//*[@id='ajaxtable']/tbody[1]/tr[@class='tr3 t_one']");
HtmlNode node = tres[10];
var an = node.SelectSingleNode("/td[2]/h3/a");
string aHref = "http://ac168.info/bt/" + an.Attributes["href"].Value;
string aText = an.InnerHtml;
hc.SetUrl(aHref);
res = hc.GetHtml();
tres = res.SelectNodes("//*[@id='read_tpc']/img");
string imgUrl = tres[0].Attributes["src"].Value;
HttpCore hcImg = new HttpCore();
hcImg.SetUrl(imgUrl);
hcImg.CurrentHttpItem.ResultType = ResultType.Byte;
//得到HTML代码
HttpResult result = hcImg.CurrentHttpHelper.GetHtml(hcImg.CurrentHttpItem);
if (result.StatusCode == System.Net.HttpStatusCode.OK)
{
//表示访问成功，具体的大家就参考HttpStatusCode类
}
//表示StatusCode的文字说明与描述
string statusCodeDescription = result.StatusDescription;
//把得到的Byte转成图片
Image img = result.ResultByte.ByteArrayToImage();
img.Save("f://imgs/" + imgUrl.Split('/').Last());
Console.WriteLine(tres);
Console.ReadLine();
RequestBase rb = new RequestBase();
rb.BindLocalCache("intc", s => GetSourceCacheInt(s[0].TryChangeType(0)), 3);
rb.BindLocalCache("person", s => GetSourceCachePerson(), null);
Stopwatch st0 = new Stopwatch();
st0.Start();
rb.BindLocalCache("ps", s => GetSourceCacheListPerson(), null);
st0.Stop();
Console.WriteLine("实例化1千万个对象，耗时【{0}】毫秒", st0.ElapsedMilliseconds);
Stopwatch st1 = new Stopwatch();
st1.Start();
for (int i = 0; i < 10000000; i++)
{
rb.BindLocalCache("ps", s => GetSourceCacheListPerson(), null);
}
st1.Stop();
Console.WriteLine("绑定1千万次本地缓存，耗时【{0}】毫秒", st1.ElapsedMilliseconds);
var t1 = rb.GetLocalCache("intc");
var t2 = rb.GetLocalCache("person");
var t3 = rb.GetLocalCache<Person>("person");
var t4 = rb.LocalCache.person;
var t5 = rb.LocalCache.intc;
Stopwatch st = new Stopwatch();
st.Start();
for (int i = 0; i < 10000000; i++)
{
var t = rb.LocalCache.ps;
var tt = rb.GetLocalCache<List<Person>>("ps");
}
st.Stop();
Console.WriteLine("读取2千万次本地缓存，耗时【{0}】毫秒", st.ElapsedMilliseconds);
}
private List<CaoQunMainItem> GetCaoQunMainItems()
{
List<CaoQunMainItem> res = new List<CaoQunMainItem>();
HttpCore hc = new HttpCore();
hc.SetUrl(CaoQunUrl);
string mainHtml = hc.GetHtml();
HtmlNodeCollection mainItems = mainHtml.SelectNodes("//*[@id='ajaxtable']/tbody[1]/tr[@class='tr3 t_one']");
if (mainItems == null)
{
return new List<CaoQunMainItem>();
}
foreach (HtmlNode mainItem in mainItems)
{
var an = mainItem.SelectSingleNode("/td[2]/h3/a");
string infoUrl = "http://ac168.info/bt/" + an.Attributes["href"].Value;
string title = an.InnerText;
CaoQunMainItem temp = new CaoQunMainItem
{
CaoQunDetailItem = new CaoQunDetailItem(),
InfoUrl = infoUrl,
Title = title,
Url = CaoQunUrl
};
res.Add(temp);
}
return res;
}
