static void Main(string[] args)
{
List <string> coinTypes = new List <string>()
{
"btcv/btc",
"btcv/usdt"
};
var type = "btcv/btc";
CoinRealCrawler crc = new CoinRealCrawler();
//Console.WriteLine(crc.Start(type));
CoinBeneCrawler cbc = new CoinBeneCrawler();
//Console.WriteLine(cbc.Start(type));
AppLiquidCrawler alc = new AppLiquidCrawler();
//Console.WriteLine(alc.Start(type));
CrawlerTool ct = new CrawlerTool();
ct.Loop(type, 1, cbc, crc, alc);
//FacebookTool ft = new FacebookTool();
//ft.Start();
Console.ReadLine();
}
public override async Task Execute(IJobExecutionContext context)
{
//获取所有的商品价格
var pList = await _productRepository.GetAllListAsync(u => !string.IsNullOrEmpty(u.LinkAddr));
//通过爬虫来获取所有的数据
pList.ForEach(async u =>
{
//如果商品编号为空,通过地址来获取商品的对应的编号
if (string.IsNullOrEmpty(u.Skuid))
{
CrawlerOptions crawlerOptions = new CrawlerOptions()
{
Url = u.LinkAddr,
CssSelectors = new List <SelectorOptions>()
{
new SelectorOptions("skuid", "#detail > div.tab-con > div:nth-child(1) > div.p-parameter > ul.parameter2.p-parameter-list > li:nth-child(2)")
}
};
var crawler = await CrawlerTool.GetResultAsync(crawlerOptions);
string skuid = crawler["skuid"].FirstOrDefault()?.Attributes["title"];
if (!string.IsNullOrEmpty(skuid))
{
u.Skuid = skuid;
await GetAndSetPrice(skuid, u);
}
}
else
{
await GetAndSetPrice(u.Skuid, u);
}
});
}
/// <summary>
/// 通过请求来爬取doc网页的数据信息
/// </summary>
/// <returns></returns>
public async Task <AjaxResponse> GetInfoByUrl()
{
string url = Request.Query["url"];
var content = await HttpTools.GetStringAsync(url);
//1:创建crawlerOption
CrawlerOptions options = new CrawlerOptions()
{
Content = content,
CssSelectors = new List <SelectorOptions>()
{
new SelectorOptions("title", "body > div:nth-child(7) > div > div.itemInfo-wrap > div.sku-name"),
new SelectorOptions("desc", "#detail > div.tab-con > div:nth-child(1) > div.p-parameter > ul.parameter2.p-parameter-list"),
new SelectorOptions("skuid", "#detail > div.tab-con > div:nth-child(1) > div.p-parameter > ul.parameter2.p-parameter-list > li:nth-child(2)"),
new SelectorOptions("imgs", "#spec-list > ul > li > img"),
}
};
var result = await CrawlerTool.GetResultAsync(options);
var title = result["title"].FirstOrDefault().Text.Trim();
var desc = result["desc"].FirstOrDefault().OutHtml;
//商品编号
var skuid = result["skuid"].FirstOrDefault()?.Attributes["title"];
//下载image
var imgs = result["imgs"];
List <string> imgList = new List <string>();
if (imgs.Count > 0)
{
imgs.ForEach(u =>
{
//1.得到小图片
var imgSrc = u.Attributes["src"];
//2.得到大图片
var bigImg = "https:" + imgSrc.Replace("n5/s54x54_jfs", "n1/s450x450_jfs");
//3.下载图片
imgList.Add(bigImg);
});
}
return(await Task.FromResult(new AjaxResponse()
{
Result = new { Title = title, Desc = desc, Imgs = imgList, Skuid = skuid }
}));
}
