protected override void Run()
{
BidWebsiteSpider sp = new BidWebsiteSpider();
var config = new BidSourceConfig
{
ListUrl = "http://www.chinabidding.com/zbzx.jhtml?method=outlineOne&type=biddingProjectGG&channelId=205"
,
DetailUrlPattern = @"http://www.chinabidding.com/zbzx-detail-\d+.html"
,
TitleXpath = "/html/body/div/div[2]/div[2]/div[1]/div/h2"
,
ContentXpath = "/html/body/div/div[2]/div[2]/div[1]/div/div[2]"
,
PubishDateXpath = "/html/body/div/div[2]/div[2]/div[1]/div/div[1]"
,
PubishDatePattern = @"(\d{4}\.\d{2}\.\d{2})"
};
//BidSourceConfigBLL configService = new BidSourceConfigBLL();
//configService.Insert(config);
var bidList = sp.DownLoadBids(config);
BidBLL bidService = new BidBLL();
foreach (var entity in bidList)
{
bidService.Insert(entity);
}
}
static void Main(string[] args)
{
IBidSourceConfigBLL configService = new BidSourceConfigBLL();
int pageCount, recordCount;
int pageIndex = 1;
do
{
var configList = configService.GetList(null, pageIndex, 20, out pageCount, out recordCount);
foreach (var config in configList)
{
if (config != null && !string.IsNullOrWhiteSpace(config.ListUrl))
{
//ThreadPool.QueueUserWorkItem(d =>
//{
// BidWebsiteSpider sp = new BidWebsiteSpider();
// var bidList = sp.DownLoadBids((BidSourceConfig)d);
// BidBLL bidService = new BidBLL();
// foreach (var entity in bidList)
// {
// bidService.Insert(entity);
// }
//}, config);
BidWebsiteSpider sp = new BidWebsiteSpider();
var bidList = sp.DownLoadBids(config);
BidBLL bidService = new BidBLL();
foreach (var entity in bidList)
{
bidService.Insert(entity);
}
//bidService.CreateLuceneIndex(bidList);
}
}
}while (pageIndex > pageCount);
Console.ReadKey();
}
