public RootMinPrice GetPriceInFare(SearchInputService input)
{
try
{
CrawlerHelper helper = new CrawlerHelper();
CookieCollection cookieCollection = helper.getCookieCollection("https://ameliaweb5.intelisys.ca/VietJet/ameliapost.aspx?lang=vi", "POST", getPostdata(input));
HtmlDocument htmld = helper.getDocument("https://ameliaweb5.intelisys.ca/VietJet/ameliapost.aspx?lang=vi", "POST", getPostdata2(input), cookieCollection);
//File.WriteAllText("D:",htmld.DocumentNode.InnerHtml);
HtmlNode nodeTemp = htmld.DocumentNode.SelectSingleNode("//div[@id='vvDepDiv']/table");
RootMinPrice root = new RootMinPrice();
root.departurePrices = new OutInBoundPrices();
root.departurePrices.prices = GetListPrices(nodeTemp, input.DepartTime);
if (input.IsRoundTrip)
{
root.arrivePrices = new OutInBoundPrices();
nodeTemp = htmld.DocumentNode.SelectSingleNode("//div[@id='vvRetDiv']/table");
root.arrivePrices.prices = GetListPrices(nodeTemp, input.ReturnTime);
}
return root;
}
catch
{
return null;
}
}
public Root CreateXmlDocument(SearchInputService input)
{
try
{
CrawlerHelper helper = new CrawlerHelper();
Root root = new Root();
root.Departure = input.DepartureCode;
root.Arrival = input.ArrivalCode;
CookieCollection cookieCollection = helper.getCookieCollection("https://ameliaweb5.intelisys.ca/VietJet/ameliapost.aspx?lang=vi", "POST", getPostdata(input));
HtmlDocument htmld = helper.getDocument("https://ameliaweb5.intelisys.ca/VietJet/ameliapost.aspx?lang=vi", "POST", getPostdata2(input), cookieCollection);
root.outBound = new OutInBound();
root.inBound = new OutInBound();
root.AirLine = AirLineName.vietJetAir;
HtmlNodeCollection collection;
try
{
collection = htmld.DocumentNode.SelectNodes("//div[@id='toDepDiv']/div[@id='travOpsMain']/table[1]/tr[7]/td/table[1]/tr");
root.outBound.Items = getitems(input.DepartTime,input, collection);
}
catch { }
//chieu ve
if (input.IsRoundTrip)
{
collection = htmld.DocumentNode.SelectNodes("//div[@id='toRetDiv']/div[@id='travOpsMain']/table[1]/tr[7]/td/table[1]/tr");
root.inBound.Items = getitems(input.ReturnTime,input, collection);
}
return root;
}
catch
{
return null;
}
}
public RootMinPrice GetPriceInFare(SearchInputService input)
{
CrawlerHelper helper = new CrawlerHelper();
CookieCollection c = helper.getCookieCollection("http://booknow.jetstar.com/lowfarefinder.aspx", "GET", getGetData(input));
HtmlDocument htmld = helper.getDocument("http://booknow.jetstar.com/lowfarefinder.aspx", "GET", getGetData(input), c);
HtmlNodeCollection nodeTemp = htmld.DocumentNode.SelectNodes("//div[@id='departing-flight']/div/div[@class='low-fare-selector']/ul/li");
RootMinPrice root = new RootMinPrice();
root.departurePrices = new OutInBoundPrices();
root.departurePrices.prices = GetListPrices(nodeTemp);
root.arrivePrices = new OutInBoundPrices();
nodeTemp = htmld.DocumentNode.SelectNodes("//div[@id='returning-flight']/div/div[@class='low-fare-selector']/ul/li");
root.arrivePrices.prices = GetListPrices(nodeTemp);
return root;
}
public Root CreateXmlDocument(SearchInputService input)
{
CrawlerHelper helper = new CrawlerHelper();
Root root = new Root();
root.Departure = input.DepartureCode;
root.Arrival = input.ArrivalCode;
root.AirLine = AirLineName.jetstar;
HtmlDocument htmld = helper.getDocument("http://booknow.jetstar.com/Search.aspx?culture=vi-VN", "POST", getPosData(input), null);
//chua 2 bang ve chieu di va ve
HtmlNodeCollection collection = htmld.DocumentNode.SelectNodes("//table[@class='domestic']");
#region Depart
try
{
root.outBound = new OutInBound();
List<Item> items = new List<Item>();
HtmlNode nodeDepart = collection[0];
//cac dong chu ban ghi chuyen bay
var collectiontr = nodeDepart.SelectNodes("tbody/tr").Where(i => i.Attributes.Count == 0).ToList();
root.outBound.Items = getItems(collectiontr,input);
}
catch { }
#endregion Depart
#region Return
try
{
root.inBound = new OutInBound();
HtmlNode nodeReturn = collection[1];
//cac dong chu ban ghi chuyen bay
var collectiontr = nodeReturn.SelectNodes("tbody/tr").Where(i => i.Attributes.Count == 0).ToList();
root.inBound.Items = getItems(collectiontr,input);
}
catch { }
#endregion Return
return root;
}
public Tigerair()
{
crawlerHelper = new CrawlerHelper();
}
//补充 还不完整的信息
private async Task FillDetail(DishDetailsRawDataItem detail)
{
IHtmlDocument doc = await CrawlerHelper.GetDocumentASync($"https:{detail.Url}");
//大图
var img = doc.QuerySelector(".J_photo img"); // 找出大图url
detail.BigImageUrl = img?.GetAttribute("src");
// 分类
var tips = doc.QuerySelectorAll(".recipeTip.mt16");
if (tips != null && tips.Length >= 3)
{
var links = tips[2].GetElementsByTagName("a");
StringBuilder sb = new StringBuilder();
foreach (var a in links)
{
sb.Append(a.TextContent);
sb.Append(",");
}
detail.Tags = sb.ToString();
}
// 配比bom
var particulars = doc.QuerySelectorAll(".particulars");
if (particulars != null && particulars.Length >= 2)
{
var boms = particulars.Select(p =>
{
var bom = new DishBomRawData();
var bis = p.GetElementsByTagName("li").Select(li =>
{
var spans = li.Children;
if (spans != null && spans.Length >= 2)
{
var bi = new BomItem()
{
EnglishName = CrawlerHelper.GetUrlLast(spans[0].FirstElementChild?.GetAttribute("href")),
FoodMaterialName = spans[0].GetElementsByTagName("b").FirstOrDefault()?.TextContent,
Use = spans[1].TextContent,
};
return(bi);
}
else
{
return(new BomItem());
}
}
);
bom.AddRange(bis);
return(bom);
}).ToList();
if (boms != null && boms.Count >= 2)
{
detail.DishBom = boms[0];
detail.AuxDishBom = boms[1];
}
}
// 口味等
var specs = doc.QuerySelectorAll(".recipeCategory_sub_R.mt30.clear li").Select(li =>
{
return(li.GetElementsByTagName("a").FirstOrDefault()?.TextContent);
}).ToList();
if (specs != null && specs.Count >= 4)
{
detail.Taste = specs[0];
detail.Technology = specs[1];
detail.CookTime = specs[2];
detail.Difficulty = specs[3];
}
// 步骤
var steps = doc.QuerySelectorAll(".recipeStep li").Select(li =>
{
string url = li.GetElementsByTagName("img").FirstOrDefault()?.GetAttribute("src");
string content = li.LastElementChild?.TextContent;
return(new CookeyItem()
{
Photo = url,
Content = content
});
}).ToList();
detail.Cookery = new CookeryRawData(steps);
}
static void Main(string[] args)
{
ConsoleInitial();
LinkBLL bllLink = new LinkBLL();
ArticleBLL bllArticle = new ArticleBLL();
linkList = bllLink.SelectByStatus(100, Link.StatusAttribute.IsUse);
if (linkList.Count <= 0)
{
Link link = new Link();
link.Url = BaseUrl;
link.Depth = 0;
bllLink.AddUrl(BaseUrl, 0);
linkList.Add(link);
}
int isUseLinkIndex = linkList.Count - 1;
while (linkList.Count > 0)
{
Link link = linkList[isUseLinkIndex];
//添加加载记录,修改成不能用（下次不查询）
//Loaded.Add(url, depth);
bllLink.UpdateStatusByNotUse(link.Url);
HttpHelper.HttpItem httpItem = new HttpHelper.HttpItem();
httpItem.URL = link.Url;
httpItem.Method = "GET";
httpItem.Encoding = Encoding.UTF8;
string html = HttpHelper.GetHtml(httpItem);
CrawlerHelper crawlerHelper = new CrawlerHelper(html);
string[] links = crawlerHelper.GetLinks;
AddUrls(links, link.Depth + 1, BaseUrl);
Console.WriteLine(string.Format("需加载{0},已加载{1}", bllLink.SelectCountByStatusIsUse(), bllLink.SelectCountByStatusNotUse()));
//删除 当前需要加载的
//Unload.Remove(url);
isUseLinkIndex--;
if (isUseLinkIndex <= -1)
{
linkList = bllLink.SelectByStatus(100, Link.StatusAttribute.IsUse);
isUseLinkIndex = linkList.Count - 1;
}
}
//加载需要下载的链接
linkList = bllLink.SelectByIsUseDownload(100, ArticleSuffix);
int isDownloadIndex = linkList.Count - 1;
while (linkList.Count > 0)
{
Link link = linkList[isDownloadIndex];
Console.WriteLine("----{0}=={1}-----", isDownloadIndex, link.Id);
HttpHelper.HttpItem httpItem = new HttpHelper.HttpItem();
httpItem.URL = link.Url;
httpItem.Method = "GET";
httpItem.Encoding = Encoding.UTF8;
string html = HttpHelper.GetHtml(httpItem);
CrawlerHelper crawlerHelper = new CrawlerHelper(html);
Article article = crawlerHelper.GetArticle;
if (article.Title != string.Empty && article.Content != string.Empty)
{
article.ArticleUrl = link.Url;
int artInsertId = bllArticle.InsertByLinkId(article, link.Id);
if (artInsertId > 0)
{
Console.WriteLine("添加成功-----{0}", article.Title);
}
}
else
{
bllLink.UpdateIsDownloadById(link.Id);
}
isDownloadIndex--;
if (isDownloadIndex <= -1)
{
linkList = bllLink.SelectByIsUseDownload(100, ArticleSuffix);
isDownloadIndex = linkList.Count - 1;
}
}
Console.ReadKey();
Console.WriteLine("按任意键退出...");
Console.ReadKey();
}
public AirAsia()
{
crawlerHelper = new CrawlerHelper();
}
public Abacus()
{
crawlerHelper = new CrawlerHelper();
}
