public SelectNodes ( string xpath ) : HtmlAgilityPack.HtmlNodeCollection | ||
xpath | string | The XPath expression. |
return | HtmlAgilityPack.HtmlNodeCollection |
private EuroMillionsResult ParseResultSection(HtmlNode section) { var date = DateTime.ParseExact(section.SelectSingleNode(".//div[@class = 'floatLeft']/a").InnerText, "dd/MM/yyyy", CultureInfo.InvariantCulture); var balls = section.SelectNodes(".//td[@class = 'euro-ball-s']").Select(x => Convert.ToInt32(x.InnerText)); var bonusBalls = section.SelectNodes(".//td[@class = 'euro-lucky-star-s']").Select(x => Convert.ToInt32(x.InnerText)); return new EuroMillionsResult(date, 0, balls.ToList(), bonusBalls.ToList()); }
public bool SetValue(HtmlNode n, string value) { if (n is HtmlNode && n.Name == "select") { foreach (HtmlNode o in n.SelectNodes("option")) { o.SetAttributeValue("selected", o.GetAttributeValue("value", "").Equals(value) ? "selected" : ""); } return true; } if (n is HtmlNode && n.Name == "input") { switch (n.GetAttributeValue("type", "")) { case "radio": n.SetAttributeValue("checked", n.GetAttributeValue("value", "").Equals(value) ? "checked" : ""); break; default: n.SetAttributeValue("value", value); break; } n.SetAttributeValue("value", value); return true; } return false; }
internal static List<ChannelItemInfo> ParseNode(HtmlNode node, bool abroadOnly) { var items = new List<ChannelItemInfo>(); if (node == null) return items; var playableArticles = node.SelectNodes(".//article[contains(@class, 'playJsInfo-Core') or contains(@class, 'slick_item')]"); if (playableArticles != null) foreach (var article in playableArticles) { var playable = ParsePlayableArticle(article, abroadOnly); if (playable != null) items.Add(playable); } var folderArticles = node.SelectNodes(".//article[not(contains(@class, 'playJsInfo-Core') or contains(@class, 'slick_item'))]"); if (folderArticles != null) foreach (var article in folderArticles) { var folder = ParseFolderArticle(article); if (folder != null) items.Add(folder); } return items; }
protected void Page_Load(object sender, EventArgs e) { string heads = @"Accept: application/json, text/javascript, */* q=0.01 " + @"Accept-Encoding: gzip, deflate " + @"Accept-Language: zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2 " + @"Connection: keep-alive " + @"Cookie: s_ViewType=10; _lxsdk_cuid=1729cd29d3dc8-04d80d1c3b31398-4c302c7d-144000-1729cd29d3ec8; _lxsdk=1729cd29d3dc8-04d80d1c3b31398-4c302c7d-144000-1729cd29d3ec8; _hc.v=6c48a318-c117-5df7-478a-f0f694f1570e.1591768948; Hm_lvt_602b80cf8079ae6591966cc70a3940e7=1591768950,1591788446; _lxsdk_s=1729dfc18eb-4f6-3ef-94c%7C%7C19; Hm_lpvt_602b80cf8079ae6591966cc70a3940e7=1591788446 " + @"Host: catdot.dianping.com " + @"Referer: http:/www.dianping.com/search…/0_%E8%8B%B1%AF%AD%E5%9F%B9%AE " + @"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0"; string url = @"http://www.dianping.com/search/keyword/1/0_%E8%8B%B1%AF%AD%E5%9F%B9%AE"; ClassHttpRequestClient s = new ClassHttpRequestClient(true); HtmlDocument doc = new HtmlDocument(); string content = ""; string response = s.httpPost(url, heads, content, Encoding.UTF8); HtmlAgilityPack.HtmlNodeCollection collection = doc.DocumentNode.SelectNodes("//div[@class=\"txt\"]"); StringBuilder sb = new StringBuilder(); foreach (HtmlAgilityPack.HtmlNode item in collection) { HtmlAgilityPack.HtmlNode divtit = item.SelectNodes("div[@class=\"txt\"]")[0]; HtmlAgilityPack.HtmlNode aname = divtit.SelectNodes("a[1]")[0]; HtmlAgilityPack.HtmlNode divcomment = item.SelectNodes("div[@class=\"comment\"]")[0]; HtmlAgilityPack.HtmlNode anum = divcomment.SelectNodes("a[1]")[0]; HtmlAgilityPack.HtmlNode aprice = divcomment.SelectNodes("a[2]")[0]; sb.Append(string.Format("{0}—{1}—{2}", aname.InnerText, anum.InnerText, aprice.InnerText)); } Response.Write(sb); }
private List<Movie> ParseMovieListHtml(HtmlNode htmlNode, string xPath) { try { var movies = new List<Movie>(); var hnc = htmlNode.SelectNodes(xPath); var hnc2 = htmlNode.SelectNodes("//div/a/img"); if (hnc.Count < 1) return null; for (int i = 0; i < hnc.Count; i++) { var node1 = hnc2[i]; var node = hnc[i]; var movie = new Movie(); var hac = node1.Attributes; movie.Grade =node.SelectSingleNode("div[@class='fm-movie-desc']/div/span[@class='fm-rating']").InnerText.Replace("\n", "").RemoveSpace().Trim(); movie.Image = hac[0].Value.Replace("-poster100","").Trim(); movie.Name = node.SelectNodes("div[@class='fm-movie-desc']/div")[0].InnerText.Replace("\n","").RemoveSpace().Trim(); movie.Director =node.SelectNodes("div[@class='fm-movie-desc']/div")[2].InnerText.Replace("\n", "").RemoveSpace().Trim(); movie.Story = node.SelectNodes("div[@class='fm-movie-desc']/div")[3].InnerText.Replace("\n", "").RemoveSpace().Trim(); movie.Actor =node.SelectNodes("div[@class='fm-movie-desc']/div")[4].InnerText.Replace("\n","").RemoveSpace().Trim(); movie.DetailUrl = node.SelectSingleNode("div[@class='fm-movie-cover']/a").Attributes["href"].Value.Trim(); movies.Add(movie); } return movies; } catch (Exception e) { throw e; } }
public YoutubeVideoEntry(HtmlNode node) { var url_node = node.SelectNodes(".//a[@href]"); if (url_node != null) { var url_value = url_node.FirstOrDefault().Attributes["href"].Value; var splitIndex = url_value.IndexOf("&"); if (splitIndex > 0) { url = "http://www.youtube.com" + url_value.Substring(0, splitIndex); } } var title_node = node.SelectNodes(".//span[contains(@class, 'video-title')]"); if (title_node != null) title = title_node.FirstOrDefault().InnerText; if (!String.IsNullOrEmpty(title)) title = title.Trim(); var img_node = node.SelectNodes(".//img[@src]"); if (img_node != null) imageUrl = "http:" + img_node.FirstOrDefault().Attributes["src"].Value; }
protected override string retrieveTitle(HtmlNode node) { string title = node.SelectNodes("//div[contains(@class, 'entry-content')]//strong" + "|//div[contains(@class, 'entry-content')]//b")?.First()?.InnerText ?? ""; node.SelectNodes("//div[contains(@class, 'entry-content')]//strong" + "|//div[contains(@class, 'entry-content')]//b")?.First()?.Remove(); return title; }
/// <summary> /// Run xpath from html or node /// </summary> private List<KeyValuePair<string, object>> run(HtmlNode node) { Factory.Instance.iInfo(string.Format("Running xpathSingle id : {0}", rule.id)); if (node == null) return new List<KeyValuePair<string, object>>(); //Get all attriibutes by type and save to List<KeyValuePair<string, object>> foreach (Db.xpathSingleAttributes attr in rule.attributes) { object val = null; if (attr.getType == Db.xpathSingleAttributesGetType.nodeCollection) val = node.SelectNodes(attr.xpath); else if (attr.getType == Db.xpathSingleAttributesGetType.count) { HtmlNodeCollection c = node.SelectNodes(attr.xpath); if (c != null) val = c.Count.ToString(); } else { string val2 = string.Empty; HtmlNode n = node.SelectSingleNode(attr.xpath); if (n != null) { if (attr.getType == Db.xpathSingleAttributesGetType.singleNode) val = n; else { if (attr.getType == Db.xpathSingleAttributesGetType.text) val2 = n.InnerText.Trim(); if (attr.getType == Db.xpathSingleAttributesGetType.html) val2 = n.InnerHtml.Trim(); if (attr.getType == Db.xpathSingleAttributesGetType.attribute) { if (n.Attributes[attr.attributeName] != null) val2 = n.Attributes[attr.attributeName].Value; } val = postProcessResult(val2, attr); if(attr.getType != Db.xpathSingleAttributesGetType.html && attr.getType != Db.xpathSingleAttributesGetType.nodeCollection && attr.getType != Db.xpathSingleAttributesGetType.singleNode) Factory.Instance.iInfo(string.Format("{0} = {1}",attr.id,val)); } } } res.Add(new KeyValuePair<string, object>(attr.id, val)); } return res; }
public HtmlNodeCollection GetDishInfoList(HtmlNode dishTypeNode) { var baseCollectionSite = new BaseCollectionSite(PageUrl); var dishNodeList = dishTypeNode.SelectNodes(DishesPath()); if (dishNodeList == null || dishNodeList.Count <= 0) { return new HtmlNodeCollection(null); } var scripNode = dishTypeNode.SelectSingleNode(@"./../../../../..//div[@class='rec-dishes tab-item active']/div[@class='pic-list J_toggle']/ul/script"); if (scripNode != null && !string.IsNullOrWhiteSpace(scripNode.InnerText)) { var liNodeList = baseCollectionSite.BaseHtmlNodeCollection(scripNode.InnerText); if (liNodeList != null) { var dishLiList = liNodeList.SelectNodes(".//li"); if (dishLiList != null) { foreach (var dishLi in dishLiList) { dishNodeList.Add(dishLi); } } } } return dishNodeList; }
public static H.HtmlNodeCollection SelectNodesOrEmpty(this H.HtmlNode node, string xpath) { Requires.NonNull(node, nameof(node)); var nodeCollection = node.SelectNodes(xpath); return(nodeCollection ?? new H.HtmlNodeCollection(node)); }
private void ParseRegionElement(HtmlNode region) { var regionTitle = region.SelectSingleNode("h2").InnerText; foreach (var server in region.SelectNodes(".//div[@class=\"server\" or @class=\"server alt\"]")) { var serverName = server.SelectSingleNode(".//div[@class=\"server-name\"]").InnerText.Trim(); var pollCategoryValue = new PollCategoryValue(); var possibleCategoryMatch = Categories.FirstOrDefault(p => string.Compare(p.Region, regionTitle, true) == 0 && string.Compare(p.ServerCategory, serverName) == 0); if (possibleCategoryMatch == null) continue; pollCategoryValue.CategoryID = possibleCategoryMatch.PollCategoryID; pollCategoryValue.Status = PollStatusType.Unknown; pollCategoryValue.CreatedTime = DateTime.Now; foreach (var div in server.SelectNodes("div")) { if (div.OuterHtml.Contains("status-icon")) { pollCategoryValue.Status = div.OuterHtml.Contains("status-icon up") ? PollStatusType.Up : PollStatusType.Down; } } DB.InsertPollCategoryValue(pollCategoryValue); } }
public List<FormElement> GetOptions(HtmlNode htmlNode) { List<FormElement> options = new List<FormElement>(); HtmlNodeCollection nodeTags = htmlNode.SelectNodes(@".//option"); if (nodeTags != null) { foreach (HtmlNode node in nodeTags) { string id = node.GetAttributeValue("id", ""); string type = "option"; string name = node.GetAttributeValue("name", ""); string value = node.GetAttributeValue("value", ""); bool chk = node.Attributes["selected"] != null; FormElement el = new FormElement(); el.Id = id; el.Type = type; el.Name = node.NextSibling.InnerText; el.Value = value; el.Type = type; el.Checked = chk; options.Add(el); } } return options; }
private int AddSubcats(HtmlNode node, RssLink parentCat) { var subs = node.SelectNodes(".//article"); foreach (var sub in subs) { RssLink subcat = new RssLink() { ParentCategory = parentCat }; subcat.Name = HttpUtility.HtmlDecode(sub.SelectSingleNode(".//a[@title]").Attributes["title"].Value.Trim()); subcat.Url = FormatDecodeAbsolutifyUrl(parentCat.Url, sub.SelectSingleNode(".//a[@href]").Attributes["href"].Value, null, UrlDecoding.None); subcat.Thumb = getThumb(sub.SelectSingleNode(".//picture/img")); parentCat.SubCategories.Add(subcat); } var np = node.SelectSingleNode(".//a[@href and text()='More shows']"); nextPageAvailable = false; if (np != null) { string url = CreateUrl(parentCat.Url, np.Attributes["href"].Value); var npCat = new NextPageCategory() { Url = url, ParentCategory = parentCat }; parentCat.SubCategories.Add(npCat); } parentCat.SubCategoriesDiscovered = true; return(parentCat.SubCategories.Count); }
private static Nhl_Games_Rtss MapHtmlRowToModel(HtmlNode row, NhlSeasonType nhlSeasonType) { HtmlNodeCollection tdNodes = row.SelectNodes(@"./td"); Nhl_Games_Rtss model = new Nhl_Games_Rtss(); model.NhlSeasonType = nhlSeasonType; model.Date = Convert.ToDateTime(tdNodes[0].InnerText.Replace("'", "/")); model.Year = NhlModelHelper.GetSeason(model.Date).Item2; model.GameNumber = Convert.ToInt32(tdNodes[1].InnerText); model.Visitor = tdNodes[2].InnerText; model.Home = tdNodes[3].InnerText; model.RosterLink = NhlGamesRtss.ParseLinkFromTd(tdNodes[4]); model.GameLink = NhlGamesRtss.ParseLinkFromTd(tdNodes[5]); model.EventsLink = NhlGamesRtss.ParseLinkFromTd(tdNodes[6]); model.FaceOffsLink = NhlGamesRtss.ParseLinkFromTd(tdNodes[7]); model.PlayByPlayLink = NhlGamesRtss.ParseLinkFromTd(tdNodes[8]); model.ShotsLink = NhlGamesRtss.ParseLinkFromTd(tdNodes[9]); model.HomeToiLink = NhlGamesRtss.ParseLinkFromTd(tdNodes[10]); model.VistorToiLink = NhlGamesRtss.ParseLinkFromTd(tdNodes[11]); model.ShootoutLink = NhlGamesRtss.ParseLinkFromTd(tdNodes[12]); return model; }
private static void AddPackage(SteamApp app, HtmlNode packageNode) { var package = app.AddNewPackage(); var packageTitleNode = packageNode.SelectSingleNode($"//{PackageTitle}"); package.Title = packageTitleNode.InnerHtml.Replace("Buy ", "").Trim(); var priceNodes = packageNode.SelectNodes($"//div[@class='{PackagePriceXPath}']"); if (priceNodes != null) { var priceNode = priceNodes[0]; package.CurrentPrice = ParseNodeWithCurrencyToDecimal(priceNode); package.OriginalPrice = package.CurrentPrice; } else { var originalPriceNode = packageNode.SelectSingleNode($"//div[@class='{PackageOriginalPriceXPath}']"); package.OriginalPrice = ParseNodeWithCurrencyToDecimal(originalPriceNode); var discountPriceNode = packageNode.SelectSingleNode($"//div[@class='{PackageDiscountPriceXPath}']"); package.CurrentPrice = ParseNodeWithCurrencyToDecimal(discountPriceNode); } }
public IList<Flight> ParseFlights(HtmlNode documentNode) { var element = documentNode.SelectNodes("//table[@class='resultTable dealsResults']/tbody//tr[position()>1]"); IEnumerable<Flight> flights = new List<Flight>(); if (element == null) return flights.ToList(); flights = from row in element where row.HasChildNodes let departureAirport = row.ChildNodes[1].InnerText let destination = row.ChildNodes[3].InnerText let departureDate = row.ChildNodes[5].SelectSingleNode("ul/li").InnerText let returnDate = row.ChildNodes[5].SelectSingleNode("ul/li[position()>1]").InnerText let departFlightTime = row.ChildNodes[7].SelectSingleNode("ul/li/ul/li").InnerText let returnFlightTime = row.ChildNodes[7].SelectSingleNode("ul/li[position()>1]/ul/li").InnerText let noOfNights = row.ChildNodes[9].InnerText let departureAirportCode = row.ChildNodes[13].SelectSingleNode("fieldset/input[@id='depAP']").GetAttributeValue("value", "N/a") let arrivalAirportCode = row.ChildNodes[13].SelectSingleNode("fieldset/input[@id='retAP']").GetAttributeValue("value", "N/a") let seats = row.SelectSingleNode("td[@class='seatsLeft']").ChildNodes.Count > 2 ? row.SelectSingleNode("td[@class='seatsLeft']/div").InnerText : "0" select new Flight { DepartureAirport = new Airport { Code = departureAirportCode, Name = departureAirport }, ArrivalAirport = new Airport { Code = arrivalAirportCode, Name = destination }, ArrivalDate = (departureDate + " " + departFlightTime + ":00").ToFormattedDateString(), SeatsLeft = seats.ToInt32(), DepartureDate =(returnDate + " " + returnFlightTime + ":00").ToFormattedDateString(), NoOfNights = noOfNights.ToInt32() }; return flights.ToList(); }
public static void Classification(Menu menu, agi.HtmlNode node) { agi.HtmlNodeCollection divide_td = node.SelectNodes(".//td"); agi.HtmlNodeCollection check_div = divide_td[0].SelectNodes(".//div"); agi.HtmlNodeCollection check_br = divide_td[0].SelectNodes(".//br"); int count = check_br.Count; if (check_div == null) { return; } if (count > 2) { String text = divide_td[0].InnerHtml; text = text.Replace("<br>", "</div><div>"); divide_td[0].InnerHtml = text; agi.HtmlNodeCollection tmp = divide_td[0].SelectNodes(".//div"); for (int i = 0; i < tmp.Count; i++) { menu.menu.Add(tmp[i].InnerText); } } else { //menu.menu.Add(node.InnerText); for (int i = 0; i < count; i++) { menu.menu.Add(check_div[i].InnerText); } } }
private static List<Item> extractSelect(HtmlNode html, string nameSelect) { List<Item> country = new List<Item>(); var criterioBusqueda = "//select[@name='"+ nameSelect +"']"; foreach (HtmlNode item in html.SelectNodes(criterioBusqueda)) { string[] valueText = item.InnerText.Split('\n'); List<HtmlNode> values = item.Elements("option").ToList(); for (int i = 0; i < values.Count; i++) { var aux = values[i].GetAttributeValue("value", ""); if (!aux.Equals("")) { int parse = int.Parse(aux); //tener en cuenta que en el caso de la pagina ejemplo hay un elemento vacio al inicio. //puede que no pase asi con todas. string value = valueText[i + 1]; country.Add(new Item(parse, value)); } } } return country; }
private static string GetBody(HtmlNode docNode) { var paragraphs = docNode.SelectNodes("//div[@class='delfi-article-body']//p"); var text = String.Join(" ", paragraphs.Elements().Select(e => e.InnerText)); return text; }
private void SearchFromNode(HtmlNode baseNode) { var nodes = Enumerable.Empty<HtmlNode>(); if (!_html.DocumentNode.HasChildNodes) ParseHtml(); if (chkXPath.IsChecked == true) nodes = baseNode.SelectNodes(txtSearchTag.Text); else nodes = baseNode.Descendants(txtSearchTag.Text); if (nodes == null) return; listResults.Items.Clear(); foreach (var node in nodes) { var tr = new NodeTreeView { BaseNode = node }; var lvi = new ListBoxItem(); var pnl = new StackPanel(); pnl.Children.Add(new Label { Content = string.Format("id:{0} name:{1} children{2}", node.Id, node.Name, node.ChildNodes.Count), FontWeight = FontWeights.Bold }); pnl.Children.Add(tr); lvi.Content = pnl; listResults.Items.Add(lvi); } tabControl1.SelectedItem = tabSearchResults; }
/// <summary> /// Identify Macros /// </summary> /// <param name="skeleton">skeleton</param> /// <param name="pageType">page Type</param> /// <returns></returns> private void IdentifyMacros(HtmlNode skeleton, PageType pageType) { var menuProperties = new List<PropertyDTO>(); var doc = skeleton.OwnerDocument; var propertyFactory = factory.PropertyFactory; foreach (var menuNode in pageType.MacroXpaths .SelectMany(xpath => skeleton.SelectNodes(xpath)) .Where(n => n != null && n.ParentNode != null)) { var property = propertyFactory.GetNew(); menuProperties.Add(property); var propertyNode = doc.CreateTextNode(property.TemplateReference); menuNode.ParentNode.ReplaceChild(propertyNode, menuNode); } var macros = menuProperties.Select(p => new Definition { Number = p.Number, Name = p.Name, TemplateReference = p.TemplateReference, IsMacro = true }); pageType.Definitions.AddRange(macros); }
private ProductBasicData BuildProductBasicData(HtmlNode productNode) { log.DebugFormat("[BuildProductBasicData] OuterHtml= {0}.", productNode.OuterHtml); //throw new NotImplementedException(); ProductBasicData returnValue = new ProductBasicData(); //should be transfered to barcode, but last digitals of the barcode is productId returnValue.Barcode = productNode.Attributes.First(c => c.OriginalName == "DdPiD").Value; returnValue.ProductId = productNode.Attributes.First(c => c.OriginalName == "DdPiD").Value; returnValue.pbcatid = productNode.Attributes.First(c => c.OriginalName == "pbcatid").Value; returnValue.qty = productNode.Attributes.First(c => c.OriginalName == "qty").Value; returnValue.iq = productNode.Attributes.First(c => c.OriginalName == "iq").Value; returnValue.inb = productNode.Attributes.First(c => c.OriginalName == "inb").Value; //returnValue.ImageSource = productNode.SelectSingleNode("//img[@src]").Attributes.FirstOrDefault(c=> c.Name == "src").Value; returnValue.ImageSource = productNode.SelectNodes("child::*/child::div/child::img").First().Attributes.First(c => c.Name == "src").Value; returnValue.EffectivePrice = productNode.SelectSingleNode("child::*/child::div/child::div/child::div/child::span").InnerText; //returnValue.EffectivePrice = productNode.SelectSingleNode("//span[@id='spnEffectivePrice']").InnerText; //returnValue.Description = HttpUtility.HtmlDecode(productNode.SelectSingleNode("//div[@class='ProdBoxSupplierText']").InnerText); returnValue.Description = HttpUtility.HtmlDecode(productNode.SelectSingleNode("child::*/child::div/child::div/child::a").InnerText); returnValue.ProductName = HttpUtility.HtmlDecode(productNode.SelectSingleNode("child::*/child::div/child::div/child::a").InnerText); log.DebugFormat("[BuildProductBasicData] fetched product={0}.", returnValue.ToString()); return returnValue; }
private static Nhl_Players_Rtss_Skater MapHtmlRowToModel(HtmlNode row, NhlSeasonType nhlSeasonType, int year) { HtmlNodeCollection tdNodes = row.SelectNodes(@"./td"); Nhl_Players_Rtss_Skater model = new Nhl_Players_Rtss_Skater(); model.NhlSeasonType = nhlSeasonType; model.Year = year; model.Number = 0; model.Name = tdNodes[1].InnerText; model.Team = tdNodes[2].InnerText; model.Position = tdNodes[3].InnerText; model.GamesPlayed = ConvertStringToInt(tdNodes[4].InnerText); model.Hits = ConvertStringToInt(tdNodes[5].InnerText); model.BlockedShots = ConvertStringToInt(tdNodes[6].InnerText); model.MissedShots = ConvertStringToInt(tdNodes[7].InnerText); model.Giveaways = ConvertStringToInt(tdNodes[8].InnerText); model.Takeaways = ConvertStringToInt(tdNodes[9].InnerText); model.FaceoffsWon = ConvertStringToInt(tdNodes[10].InnerText); model.FaceoffsLost = ConvertStringToInt(tdNodes[11].InnerText); model.FaceoffsTaken = ConvertStringToInt(tdNodes[12].InnerText); model.FaceoffWinPercentage = Convert.ToDouble(tdNodes[13].InnerText); model.PercentageOfTeamFaceoffsTaken = Convert.ToDouble(tdNodes[14].InnerText); model.Shots = ConvertStringToInt(tdNodes[15].InnerText); model.Goals = ConvertStringToInt(tdNodes[16].InnerText); model.ShootingPercentage = Convert.ToDouble(tdNodes[17].InnerText); return model; }
private IList<Achievement> ParseSubAchievements(Achievement achievement, HtmlNode subAchievementNode) { IList<Achievement> subAchievements = new List<Achievement>(); HtmlNodeCollection achievements = subAchievementNode.SelectNodes("./li"); if (achievements != null) { foreach (HtmlNode subNode in achievements) { // TODO : Parse out achievement id if ( subNode.Attributes["onmousemove"] != null ) { Match match = parseTooltip.Match(subNode.Attributes["onmousemove"].Value); if ( match.Success ) { string subAchievementId = match.Groups["achievementid"].Value; int blizzardId = 0; int.TryParse(subAchievementId, out blizzardId); Achievement subAchievement = _service.Find(blizzardId); if (subAchievement == null) { subAchievement = new Achievement() { BlizzardID = blizzardId }; subAchievement.Name = GetValueAsString(subNode, ".//h3"); subAchievement.Description = GetValueAsString(subNode, ".//div[@class='color-tooltip-yellow']"); subAchievement.Points = GetValueAsInt32(subNode, ".//span[@class='points border-3']"); _service.Save(subAchievement); } achievement.Points = achievement.Points - subAchievement.Points; subAchievements.Add(subAchievement); } } } } return subAchievements; }
/// <summary> /// Run xpath from html or node /// </summary> public List<List<KeyValuePair<string, object>>> run(HtmlNode node) { Factory.Instance.iInfo(string.Format("Running xpathCollection id : {0}", rule.id)); HtmlNodeCollection nodes = new HtmlNodeCollection(node); HtmlNodeCollection n2 = node.SelectNodes(rule.xpath); if (n2 != null) { foreach (HtmlNode n in n2) nodes.Add(n); } //run if (node != null) { foreach (HtmlNode n in nodes) { List<KeyValuePair<string, object>> last_val = null; if (rule.xpathSingle != null) { XPathSingle xs = new XPathSingle(rule.xpathSingle, last_val); last_val = (List<KeyValuePair<string, object>>)xs.Run(n); res.Add(last_val); } } } return res; }
private static Nhl_Players_Bio_Goalie MapHtmlRowToModel(HtmlNode row, NhlSeasonType nhlSeasonType, int year) { HtmlNodeCollection tdNodes = row.SelectNodes(@"./td"); Nhl_Players_Bio_Goalie model = new Nhl_Players_Bio_Goalie(); model.NhlSeasonType = nhlSeasonType; model.Year = year; model.Number = ConvertStringToInt(tdNodes[0].InnerText); model.Name = tdNodes[1].InnerText; model.Team = tdNodes[2].InnerText; model.Position = "G"; model.DateOfBirth = Convert.ToDateTime(tdNodes[3].InnerText.Replace("'", "/")); model.BirthCity = tdNodes[4].InnerText; model.StateOrProvince = tdNodes[5].InnerText; model.BirthCountry = tdNodes[6].InnerText; model.HeightInches = ConvertStringToInt(tdNodes[7].InnerText); model.WeightLbs = ConvertStringToInt(tdNodes[8].InnerText); model.Catches = tdNodes[9].InnerText; model.Rookie = tdNodes[10].InnerText; model.DraftYear = ConvertStringToInt(tdNodes[11].InnerText); model.DraftRound = ConvertStringToInt(tdNodes[12].InnerText); model.DraftOverall = ConvertStringToInt(tdNodes[13].InnerText); model.GamesPlayed = ConvertStringToInt(tdNodes[14].InnerText); model.Wins = ConvertStringToInt(tdNodes[15].InnerText); model.Losses = ConvertStringToInt(tdNodes[16].InnerText); model.OTSOLosses = ConvertStringToInt(tdNodes[17].InnerText); model.GAA = Convert.ToDouble(tdNodes[18].InnerText); model.SavePercentage = Convert.ToDouble(tdNodes[19].InnerText); model.Shutouts = ConvertStringToInt(tdNodes[20].InnerText); return model; }
void ParseHTMLTable(Hashtable loadLanguages, Hashtable loadEntries, HtmlAgilityPack.HtmlNode node) { int row = -1; bool firstRow = true; foreach (HtmlAgilityPack.HtmlNode trNode in node.SelectNodes(".//tr")) { if (trNode.SelectNodes(".//td") == null) { continue; } row++; int i = -1; string key = ""; foreach (HtmlAgilityPack.HtmlNode tdNode in trNode.SelectNodes(".//td")) { i++; if (firstRow && row == 0) { //Language header if (i == 0) { continue; //Ignore this top-left empty cell } loadLanguages[i] = tdNode.InnerText; if (!loadEntries.ContainsKey(i)) { loadEntries[i] = new Hashtable(); } } else { //Data rows if (i == 0) { key = tdNode.InnerText; continue; } if (key == "") { continue; //Skip entries with empty keys (the other values can be used as labels) } string content = tdNode.InnerText; Hashtable hTable = (Hashtable)loadEntries[i]; if (hTable.ContainsKey(key)) { Debug.LogError("ERROR: Double key [" + key + "]"); unresolvedErrors++; } hTable[key] = content; } } firstRow = false; } }
private static string getItemPrice(HtmlNode itemPriceNode) { HtmlNodeCollection spans = itemPriceNode.SelectNodes("span"); string price = itemPriceNode.InnerText.Replace(" ", " ").TrimStart(); string result = parsePriceRecursion(spans[0].Attributes["title"].Value, price); return parsePrice(result); }
public override string ReadContent(HtmlNode node) { RemoveTags(node.SelectNodes("//*[@id='taboola-bottom-main-column']")); var content = node.SelectSingleNode("//*[@id='news-story']/article"); RemoveScripts(content); return content.InnerHtml; }
private void buttonParseHtml_Click(object sender, EventArgs e) { OpenFileDialog openFileDialog = new OpenFileDialog(); openFileDialog.Filter = "HTML File (*.html;)|*.html"; openFileDialog.Multiselect = false; if (openFileDialog.ShowDialog() == DialogResult.OK) { if (String.IsNullOrEmpty(openFileDialog.FileName)) { return; } string strHtml = String.Empty; using (StreamReader reader = new StreamReader(openFileDialog.FileName, Encoding.UTF8)) { strHtml = reader.ReadToEnd(); reader.Close(); } HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(strHtml); //加载html foreach (var err in doc.ParseErrors) { Console.WriteLine(err.Code); } //Console.WriteLine(doc.Text); HtmlAgilityPack.HtmlNode rootNode = doc.DocumentNode; //获取文档的根节点 //Console.WriteLine(rootNode.OuterHtml); #if true string xpath = @"//table"; HtmlAgilityPack.HtmlNode node = rootNode.SelectSingleNode(xpath); //获取表格 foreach (var script in node.Descendants("script").ToArray()) { script.Remove(); } foreach (var style in node.Descendants("style").ToArray()) { style.Remove(); } string innerText = node.OuterHtml; //到这里就是纯纯的表格了 var trNodes = node.SelectNodes("tr"); foreach (var trnod in trNodes) //遍历行 { var tdNodes = trnod.SelectNodes("td"); for (int i = 0; i < tdNodes.Count; i++) //遍历列 { Console.WriteLine(tdNodes[i].InnerText); } } #endif } }
public static Remark ParseRemark(HtmlNode remarkNode) { var remark = new Remark(); //发现物 var discoveryNode = remarkNode.SelectSingleNode("a[@title!='']"); if (discoveryNode!=null) { var levelNode = discoveryNode.PreviousSibling.PreviousSibling; var typeNode = levelNode.PreviousSibling.PreviousSibling; remark.DiscoveryType = Enum.Parse(typeof(DisType), typeRegex.Match(typeNode.Attributes["src"].Value).Groups["type"].Value).ToString(); remark.DiscoveryLevel = Int32.Parse(levelNode.InnerText.Substring(0, 1)); remark.DiscoveryExp = Int32.Parse(discoveryNode.Attributes["title"].Value.Remove(0,5)); remark.Discovery = discoveryNode.InnerText; } //奖励物 var awardNode = remarkNode.SelectSingleNode("span[@style='color:#804000;']"); if (awardNode != null) remark.AwardItem = awardNode.InnerText; //相关任务 var relativeNodes = remarkNode.SelectNodes("descendant::a[@style='color:#C000C0;' or @style='color:DarkBlue;']"); if (relativeNodes != null) { foreach (HtmlNode relativeNode in relativeNodes) { IList<int> questList = null; IList<string> foundNameList = null; if (relativeNode.InnerText.StartsWith("前:")) { foundNameList = remark.PreFoundName; questList = remark.PreQuestID; } else { questList = remark.FollowQuestID; } var match = questRegex.Match(relativeNode.Attributes["href"].Value); if (relativeNode.InnerText.StartsWith("前:港口-") == false) questList.Add(Int32.Parse(match.Groups["id"].Value)); else foundNameList.Add(relativeNode.InnerText.Replace("前:港口-","")); } } //接受城市 //last br next a var cityNodes = remarkNode.SelectNodes("descendant::a[@class='MisCity']"); if (cityNodes != null) { cityNodes.All(node => { if (node.InnerText == "南美开拓港" || node.InnerText == "东南亚开拓港" || node.InnerText == "掠夺地图" || node.InnerText == "沉船资讯") return true; remark.FromCityList.Add(node.InnerText); return true; }); } return remark; }
public override string ReadContent(HtmlNode node) { var content = node.SelectNodes("//div").FirstOrDefault(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("body")); RemoveScripts(content); return CleanHtml(content.InnerHtml); }
public static string getTable(HtmlNode table) { var data = ""; foreach (HtmlNode row in table.SelectNodes("tr")) { data += row.InnerText + ";"; } return data; }
public IEnumerable<HtmlNode> FindWithin(HtmlNode xmlElement) { var nodes = xmlElement.SelectNodes(xpath); if(nodes == null) { return new List<HtmlNode>(); } return nodes; }
Dictionary<decimal, decimal> ParseOrderDepthFromResult(HtmlNode table) { return table .SelectNodes("tr[position() > 1]") // Skip the header row .ToDictionary( tr => Decimal.Parse(tr.ChildNodes[0].InnerText), tr => Decimal.Parse(tr.ChildNodes[1].InnerText) ); }
public static HAP.HtmlNodeCollection SelectCollection(this HAP.HtmlNode parent, string xPath, bool doThrow) { var node = parent.SelectNodes(xPath); if (doThrow && node == null) { throw new NewsParserException(new { xPath }); } return(node ?? new HAP.HtmlNodeCollection(parent)); }
public string Translate (HtmlNode commandNode) { HtmlNodeCollection cellNodes = commandNode.SelectNodes ("td"); string command = cellNodes[0].InnerText; string target = cellNodes[1].InnerText; string value = cellNodes[2].InnerText; return Translate (command, target, value); }
public FormElement(HtmlAgilityPack.HtmlNode Node, int NodeIndex) : base(Node, NodeIndex) { HtmlNodeCollection NodesColl = Node.SelectNodes(".//input"); if (NodesColl != null) { for (int i = 0; i < NodesColl.Count; i++) { InputElements.Add(new InputElement(NodesColl[i], i)); } } NodesColl = Node.SelectNodes(".//select"); if (NodesColl != null) { for (int i = 0; i < NodesColl.Count; i++) { SelectElements.Add(new Element(NodesColl[i], i)); } } }
private void BwLoad_DoWork(object sender, DoWorkEventArgs e) { HtmlAgilityPack.HtmlWeb web = new HtmlWeb(); HtmlAgilityPack.HtmlDocument htmlDoc = web.Load("https://www.naver.com"); HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//ul[@class='ah_l']"); HtmlAgilityPack.HtmlNode[] node = bodyNode.SelectNodes(".//span[@class='ah_k']").ToArray(); foreach (HtmlNode n in node) { this.result.Add(n.InnerText); } }
private static string GetDescription(HtmlAgilityPack.HtmlNode documentNode) { string description = string.Empty; var node = documentNode.SelectNodes("//h4") .SingleOrDefault( s => s.Attributes["class"] != null && s.Attributes["class"].Value.Equals("video_top_more") && s.Attributes["id"] != null && s.Attributes["id"].Value.Equals("video_top_more")); if (node != null) { description = node.InnerText.Replace("\r", "").Replace("\n", "").Replace("\t", ""); } return(description); }
private static string GetLinkOrigin(HtmlAgilityPack.HtmlNode documentNode) { string linkOrigin = string.Empty; var node = documentNode .SelectNodes( "//meta") .SingleOrDefault( a => a.Attributes["property"] != null && a.Attributes["property"].Value.Equals("og:url") && a.Attributes["itemprop"] != null && a.Attributes["itemprop"].Value.Equals("url")); if (node != null) { linkOrigin = node.Attributes["content"].Value; } return(linkOrigin); }
private List <VideoInfo> GetVids(HtmlNode node, string parentUrl) { List <VideoInfo> videos = new List <VideoInfo>(); var vids = node.SelectNodes(".//article"); foreach (var vid in vids) { VideoInfo video = new VideoInfo(); if (vid.SelectSingleNode(".//h2[contains(@class,'h3')]") == null) { video.Title = HttpUtility.HtmlDecode(vid.SelectSingleNode(".//a[@title]").Attributes["title"].Value.Trim()); video.VideoUrl = FormatDecodeAbsolutifyUrl(parentUrl, vid.SelectSingleNode(".//a[@href]").Attributes["href"].Value, null, UrlDecoding.None); } else { video.Title = vid.SelectSingleNode(".//h2[contains(@class,'h3')]").InnerText.Trim(); video.VideoUrl = FormatDecodeAbsolutifyUrl(parentUrl, vid.SelectSingleNode(".//a[@class='teaser__link' and @href]").Attributes["href"].Value, null, UrlDecoding.None); if (vid.SelectSingleNode(".//p[contains(@class,'teaser__description')]") != null) { video.Description = vid.SelectSingleNode(".//p[contains(@class,'teaser__description')]").InnerText.Trim(); } else { video.Description = vid.SelectSingleNode(".//h3[contains(@class,'teaser__subtitle')]").InnerText.Trim(); } } var moNode = vid.SelectSingleNode(".//span[@data-month]"); var daNode = vid.SelectSingleNode(".//span[@data-date]"); if (moNode != null && daNode != null) { video.Airdate = moNode.InnerText.Trim() + ' ' + daNode.InnerText.Trim(); } video.Thumb = getThumb(vid.SelectSingleNode(".//picture/img")); videos.Add(video); } var np = node.SelectSingleNode(".//a[@href and contains(text(),'More ')]"); nextPageAvailable = false; if (np != null) { nextPageAvailable = true; nextPageUrl = CreateUrl(parentUrl, np.Attributes["href"].Value); } return(videos); }
public object Transform(Dictionary <string, object> settings, HtmlNodeNavigator nodeNavigator, List <HtmlAgilityPack.HtmlNode> logicalParents) { var ret = new StringBuilder(); var foundParent = false; var currentNode = nodeNavigator?.CurrentNode; if (logicalParents != null && logicalParents.Count >= 2) { // We will skip out immediate parent because that's the list, we need the parent of the list, which is out grandparent var grandParentNode = logicalParents[logicalParents.Count - 2]; HtmlAgilityPack.HtmlNode parentNode = grandParentNode; if (settings != null && settings.ContainsKey("_startingXPath") && ((JValue)settings["_startingXPath"]).Type == JTokenType.String) { var startingXPath = ((JValue)settings["_startingXPath"]).ToObject <string>(); var nodes = parentNode.SelectNodes(startingXPath); if (nodes != null && nodes.Count > 0) { parentNode = nodes[0]; } else { return(0); } } while (currentNode != null && currentNode != parentNode && !foundParent) { var siblingText = this.GetTextFromSiblings(currentNode, parentNode, ref foundParent); if (!string.IsNullOrEmpty(siblingText)) { ret.Append(siblingText); ret.Append(" "); } currentNode = currentNode.ParentNode; } } var text = ret.ToString().Trim(); return(text.Length); }
private void GetNextPageVideosUrl(HtmlAgilityPack.HtmlNode node) { HasNextPage = false; nextPageUrl = ""; var a_o_buttons = node.SelectNodes("//a[contains(@class, 'svtoa-button')]"); if (a_o_buttons != null) { var a_o_next_button = a_o_buttons.Where(a => (a.InnerText ?? "").Contains("Visa fler")).FirstOrDefault(); if (a_o_next_button != null) { nextPageUrl = a_o_next_button.GetAttributeValue("href", ""); nextPageUrl = HttpUtility.UrlDecode(nextPageUrl); nextPageUrl = HttpUtility.HtmlDecode(nextPageUrl); //Some urls come html encoded HasNextPage = true; } } }
private static List <string> GetTags(HtmlAgilityPack.HtmlNode documentNode) { List <string> tags = new List <string>(); var node = documentNode.SelectNodes("//div") .FirstOrDefault(c => c.Attributes["class"] != null && c.Attributes["class"].Value.Equals("tag_video")); if (node != null) { var nodeTags = node.SelectNodes(".//a") .Where(a => a.Attributes["class"] != null && a.Attributes["class"].Value.Equals("eachTag_video")) .ToList(); if (nodeTags.Any()) { tags.AddRange(nodeTags.Select(nodeTag => nodeTag.InnerText)); } } return(tags); }
void ParseHTMLTable(ref List <object> loadData, HtmlAgilityPack.HtmlNode node) { foreach (HtmlAgilityPack.HtmlNode trNode in node.SelectNodes(".//tr")) { if (trNode.SelectNodes(".//td") == null) { continue; } int i = -1; List <object> rowData = new List <object>(); foreach (HtmlAgilityPack.HtmlNode tdNode in trNode.SelectNodes(".//td")) { i++; rowData.Add(tdNode.InnerText); } loadData.Add(rowData); } }
private static string ParsePsalm(HtmlAgilityPack.HtmlNode psalmText) { var resT = string.Empty; if (psalmText != null) { resT = psalmText.InnerText; if (!psalmText.InnerText.Contains("span")) { return(resT); } foreach (HtmlNode span in psalmText.SelectNodes(".//span")) { string attributeValue = span.GetAttributeValue("style", ""); if (string.IsNullOrEmpty(attributeValue)) { resT += $"{span.InnerHtml}"; } } } return(resT); }
/// <summary> /// Selects a list of nodes matching the HtmlAgilityPack.HtmlNode.XPath expression - will not return null. /// </summary> /// <param name="node">The node.</param> /// <param name="xpath">The XPath expression.</param> /// <returns>An HtmlAgilityPack.HtmlNodeCollection containing a collection of nodes matching the HtmlAgilityPack.HtmlNode.XPath query, or an empty collection if no node matched the XPath expression.</returns> public static HtmlNodeCollection SelectSafeNodes(this HtmlNode node, XPathExpression xpath) { return(node.SelectNodes(xpath) ?? new HtmlNodeCollection(node)); }
/// <summary> /// 分析得到数据表 /// </summary> /// <param name="content"></param> /// <returns></returns> private DataTable GetDataTable(Html.HtmlDocument content, bool istech) { DataTable dt = new DataTable(); // Html.HtmlDocument document = content; Html.HtmlNode table = document.DocumentNode.SelectSingleNode("/html/body/table/tr[2]/td/table/tr/td[2]/table[2]/tr[" + (istech ? "3" : "2") + "]/td/div/div/table"); //列 DataColumn dc; dc = new DataColumn("教练员"); //姓名 dt.Columns.Add(dc); dc = new DataColumn("车型"); //姓名 dt.Columns.Add(dc); Html.HtmlNodeCollection trTime = table.SelectNodes("./tr[1]/th[1]/font/b/th"); foreach (Html.HtmlNode tr in trTime) { dc = new DataColumn(tr.InnerText); dt.Columns.Add(dc); } dc = new DataColumn("教练号"); dt.Columns.Add(dc); //数据区 Html.HtmlNodeCollection trs = table.SelectNodes("./tr[position()>1]"); DataRow dr; foreach (Html.HtmlNode tr in trs) { Html.HtmlNodeCollection tds = tr.SelectNodes("./td"); dr = dt.NewRow(); int index = 0; foreach (Html.HtmlNode td in tds) { if (index == 0) { string name = td.SelectSingleNode("./font/span/child::text()[position()=1]").InnerText; string car = td.SelectSingleNode("./font/span/span").InnerText; dr[index] = name; index++; dr[index] = car; } else { if (td.InnerText == "可预约") { dr[index] = "yunyue"; } else { if (istech && index == 2) { dr[index] = td.SelectSingleNode("./font/span").InnerText; } else { dr[index] = td.InnerText; } } } index++; } dt.Rows.Add(dr); } return(dt); }
public static ViewCounterModel ViewCounter(string data, ProgressDialogController progressController) { HtmlNode td; string text; ViewCounterModel result = new ViewCounterModel(); //data = System.IO.File.ReadAllText("data"); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); if (doc == null) { return(null); } doc.LoadHtml(data); if (doc.DocumentNode == null && doc.DocumentNode.ChildNodes == null) { return(null); } if (doc.DocumentNode.ChildNodes.Count > 1) { #region азбор секции с информацией о канале связи HtmlAgilityPack.HtmlNode jqTabsDevices = doc.DocumentNode.SelectNodes("//div[@id='jqTabsDevices']").Single(); if (jqTabsDevices == null) { return(null); } HtmlNodeCollection info = jqTabsDevices.SelectNodes("div[2]/table/tbody/tr"); if (info == null) { return(null); } // Наименование точки учета td = info[0].SelectNodes("td[2]").Single(); text = td.InnerText; result.AccountPoint = text; // Тип счетчика td = info[1].SelectNodes("td[2]").Single(); text = td.InnerText; result.CounterType = text; // Заводской номер td = info[2].SelectNodes("td[2]").Single(); text = td.InnerText.Trim(); result.CounterNumber = text; // Сетевой адрес td = info[3].SelectNodes("td[2]").Single(); text = td.InnerText; result.CounterNetworkAddress = text; // Коэффициент трансформации td = info[5].SelectNodes("td[2]").Single(); text = td.InnerText; result.Ktt = text; // Производитель td = info[6].SelectNodes("td[2]").Single(); text = td.InnerText.Trim(); result.CounterManufacturer = text; // Тип учёта td = info[7].SelectNodes("td[2]").Single(); text = td.InnerText.Trim(); result.AccountType = text; // Полное название абонента td = info[8].SelectNodes("td[2]").Single(); text = td.InnerText.Trim(); result.AbonentFullName = text; // Название абонента td = info[9].SelectNodes("td[2]").Single(); text = td.InnerText.Trim(); result.AbonentName = text; // Короткое название абонента td = info[10].SelectNodes("td[2]").Single(); text = td.InnerText.Trim(); result.AbonentShortName = text; // Подстанция td = info[11].SelectNodes("td[2]").Single(); text = td.InnerText; result.Substation = text; // Название объекта td = info[12].SelectNodes("td[2]").Single(); text = td.InnerText; result.ObjectName = text; // Название точки учета td = info[13].SelectNodes("td[2]").Single(); text = td.InnerText; result.AccountPointName = text; // Номер ТП td = info[14].SelectNodes("td[2]").Single(); text = td.InnerText; result.TP = text; // Адрес объекта td = info[15].SelectNodes("td[2]").Single(); text = td.InnerText; result.ObjectAddress = text; // Населенный пункт объекта td = info[16].SelectNodes("td[2]").Single(); text = td.InnerText; result.ObjectState = text; // Адрес абонента td = info[17].SelectNodes("td[2]").Single(); text = td.InnerText; result.AbonentAddress = text; // Фидер td = info[18].SelectNodes("td[2]").Single(); text = td.InnerText; result.Fider = text; // Номер договора td = info[19].SelectNodes("td[2]").Single(); text = td.InnerText; result.DogNumber = text; // Родительский лиц счет td = info[20].SelectNodes("td[2]").Single(); text = td.InnerText.Trim(); result.AmperParentPointId = text; // РЭС td = info[21].SelectNodes("td[2]").Single(); text = td.InnerText; result.Departament = text; // Зав. номер из расч системы td = info[22].SelectNodes("td[2]").Single(); text = td.InnerText; result.AmperCounterNumber = text; // Лицевой счет td = info[23].SelectNodes("td[2]").Single(); text = td.InnerText.Trim(); result.AmperPointId = text; // Текущий статус td = info[24].SelectNodes("td[2]").Single(); text = td.InnerText; result.Status = text; // Последний сеанс td = info[25].SelectNodes("td[2]").Single(); text = td.InnerText.Trim(); DateTime date = new DateTime(); result.LastSessionDate = DateTime.TryParse( text, System.Globalization.CultureInfo.CreateSpecificCulture("en-US"), System.Globalization.DateTimeStyles.None, out date) ? date : date; #endregion #region азбор секции с показаниями HtmlAgilityPack.HtmlNode jqTabsSingleMeterIndications = doc.DocumentNode.SelectNodes("//div[@id='jqTabsSingleMeterIndications']").Single(); info = jqTabsSingleMeterIndications.SelectNodes("table/tbody/tr/td"); if (info == null) { return(null); } IndicationViewItem ivi = new IndicationViewItem(); ivi.PreviousIndications = new Indications(); ivi.NextIndications = new Indications(); #region Парсинг int startIndex = 0; // точка td = info[startIndex++]; text = td.InnerText; ivi.AccountingPoint = text; // тип td = info[startIndex++]; text = td.InnerText; ivi.CounterType = text; // предыдущие показания T0 td = info[startIndex++]; ivi.PreviousIndications.Tarriff0 = GetIndication(td.InnerText); // предыдущие показания T1 td = info[startIndex++]; ivi.PreviousIndications.Tarriff1 = GetIndication(td.InnerText); // предыдущие показания T2 td = info[startIndex++]; ivi.PreviousIndications.Tarriff2 = GetIndication(td.InnerText); // предыдущие показания T3 td = info[startIndex++]; ivi.PreviousIndications.Tarriff3 = GetIndication(td.InnerText); // предыдущие показания T4 td = info[startIndex++]; ivi.PreviousIndications.Tarriff4 = GetIndication(td.InnerText); // предыдущие показания достоверность td = info[startIndex++]; text = td.InnerText; ivi.PreviousIndications.DataReliability = text; // текущие показания T0 td = info[startIndex++]; ivi.NextIndications.Tarriff0 = GetIndication(td.InnerText); // текущие показания T1 td = info[startIndex++]; ivi.NextIndications.Tarriff1 = GetIndication(td.InnerText); // текущие показания T2 td = info[startIndex++]; ivi.NextIndications.Tarriff2 = GetIndication(td.InnerText); // текущие показания T3 td = info[startIndex++]; ivi.NextIndications.Tarriff3 = GetIndication(td.InnerText); // текущие показания T4 td = info[startIndex++]; ivi.NextIndications.Tarriff4 = GetIndication(td.InnerText); // предыдущие показания достоверность td = info[startIndex++]; text = td.InnerText; ivi.NextIndications.DataReliability = text; // разница td = info[startIndex++]; ivi.Difference = GetIndication(td.InnerText); #endregion result.IndicationViewItem = ivi; #endregion } return(result); }
public static ViewDeviceModel ViewDevice(string data, ProgressDialogController progressController) { ViewDeviceModel result = new ViewDeviceModel(); try { HtmlNode td; string text; //data = System.IO.File.ReadAllText("data"); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); if (doc == null) { return(null); } doc.LoadHtml(data); if (doc.DocumentNode == null && doc.DocumentNode.ChildNodes == null) { return(null); } if (doc.DocumentNode.ChildNodes.Count > 1) { #region азбор секции с информацией о канале связи HtmlAgilityPack.HtmlNode jqTabsDevices = doc.DocumentNode.SelectNodes("//div[@id='jqTabsDevices']").Single(); if (jqTabsDevices == null) { return(null); } HtmlNodeCollection sessionInformation = jqTabsDevices.SelectNodes("div[2]/div/table/tbody/tr"); if (sessionInformation == null) { return(null); } result.Session = new SessionInformation(); // производитель модема td = sessionInformation[2].SelectNodes("td[2]").Single(); text = td.InnerText; result.Session.ModemManufacturer = text; // Модель устройства td = sessionInformation[3].SelectNodes("td[2]").Single(); text = td.InnerText; result.Session.Model = text; // описание td = sessionInformation[4].SelectNodes("td[2]").Single(); text = td.InnerText; result.Session.Description = text; // статус td = sessionInformation[6].SelectNodes("td[2]").Single(); text = td.InnerText; result.Session.CurrentStatus = text; // сеанс td = sessionInformation[7].SelectNodes("td[2]").Single(); text = td.InnerText; DateTime date = new DateTime(); result.Session.LastSessionDate = DateTime.TryParse(text, out date) ? date : date; #endregion #region азбор секции с показаниями HtmlAgilityPack.HtmlNode jqTabsBalances = doc.DocumentNode.SelectNodes("//div[@id='jqTabsBalances']").Single(); if (jqTabsBalances == null) { return(null); } HtmlNodeCollection counters = jqTabsBalances.SelectNodes("table/tbody/tr"); if (counters == null) { return(null); } result.CountersIndications = new List <IndicationViewItem>(); byte startIndex = 0; for (int i = 0; i < counters.Count; i++) { HtmlNodeCollection hnc = counters[i].SelectNodes("td"); startIndex = 0; IndicationViewItem ivi = new IndicationViewItem(); ivi.PreviousIndications = new Indications(); ivi.NextIndications = new Indications(); #region Парсинг // точка td = hnc[startIndex++]; text = td.InnerText; ivi.AccountingPoint = text; // тип td = hnc[startIndex++]; text = td.InnerText; ivi.CounterType = text; // предыдущие показания T0 td = hnc[startIndex++]; ivi.PreviousIndications.Tarriff0 = GetIndication(td.InnerText); // предыдущие показания T1 td = hnc[startIndex++]; ivi.PreviousIndications.Tarriff1 = GetIndication(td.InnerText); // предыдущие показания T2 td = hnc[startIndex++]; ivi.PreviousIndications.Tarriff2 = GetIndication(td.InnerText); // предыдущие показания T3 td = hnc[startIndex++]; ivi.PreviousIndications.Tarriff3 = GetIndication(td.InnerText); // предыдущие показания T4 td = hnc[startIndex++]; ivi.PreviousIndications.Tarriff4 = GetIndication(td.InnerText); // предыдущие показания достоверность td = hnc[startIndex++]; text = td.InnerText; ivi.PreviousIndications.DataReliability = text; // текущие показания T0 td = hnc[startIndex++]; ivi.NextIndications.Tarriff0 = GetIndication(td.InnerText); // текущие показания T1 td = hnc[startIndex++]; ivi.NextIndications.Tarriff1 = GetIndication(td.InnerText); // текущие показания T2 td = hnc[startIndex++]; ivi.NextIndications.Tarriff2 = GetIndication(td.InnerText); // текущие показания T3 td = hnc[startIndex++]; ivi.NextIndications.Tarriff3 = GetIndication(td.InnerText); // текущие показания T4 td = hnc[startIndex++]; ivi.NextIndications.Tarriff4 = GetIndication(td.InnerText); // предыдущие показания достоверность td = hnc[startIndex++]; text = td.InnerText; ivi.NextIndications.DataReliability = text; // разница td = hnc[startIndex++]; ivi.Difference = GetIndication(td.InnerText); #endregion result.CountersIndications.Add(ivi); } #endregion #region Качество показаний if (result.QualityIndications == null) { result.QualityIndications = new List <QualityIndications>(); } HtmlNodeCollection indicationsQualityMonths = doc.DocumentNode.SelectNodes("//table[contains(@class,'tableQualityIndications')]"); if (indicationsQualityMonths != null) { int monthsCount = indicationsQualityMonths.Count; for (int monthIndex = 0; monthIndex < monthsCount; monthIndex++) { QualityIndications qi = new QualityIndications(); HtmlNode m = indicationsQualityMonths[monthIndex].SelectNodes("thead/tr[1]/th[2]").Single(); qi.Period = m == null ? "???" : m.InnerText; qi.PointsData = ParseMonthQualityIndications(indicationsQualityMonths[monthIndex].SelectNodes("tbody/tr")); result.QualityIndications.Add(qi); } } #endregion } } catch (Exception ex) { //TODO: Добавить логирование _logger?.Error(ex); return(null); } return(result); }
static void eventCrawler() { try { //Khai báo đường dẫn URL web cần lấy nội dung HTML for (int i = 0; i < 100; i++) { string _url = "https://www.adayroi.com/thuc-pham-r591?p="; HtmlAgilityPack.HtmlWeb htmlWeb = new HtmlAgilityPack.HtmlWeb(); htmlWeb.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0"; HtmlAgilityPack.HtmlDocument htmlDoc = htmlWeb.Load(_url + i); string _html = htmlDoc.DocumentNode.InnerHtml; _html = HttpUtility.HtmlDecode(_html); HtmlAgilityPack.HtmlNode _nodThreads = htmlDoc.DocumentNode.SelectSingleNode(@"//div[@class='row body-list-item']"); HtmlAgilityPack.HtmlNodeCollection nodChuDe = _nodThreads.SelectNodes(@"div"); //Khai báo biến tạm để hiện thị kết quả string ketqua = ""; //Duyet qua các nod nodChuDe vừa select được foreach (var n in nodChuDe) {//data-brand-id data-category-id data-product-item-id data-merchant-id string brand_id = n.SelectSingleNode("div").Attributes["data-brand-id"].Value.ToString().Trim(); string category_id = n.SelectSingleNode("div").Attributes["data-category-id"].Value.ToString().Trim(); string product_item_id = n.SelectSingleNode("div").Attributes["data-product-item-id"].Value.ToString().Trim(); string merchant_id = n.SelectSingleNode("div").Attributes["data-merchant-id"].Value.ToString().Trim(); string urlImage = n.SelectSingleNode("div/div[1]/span/a/img").Attributes["data-other-src"].Value.ToString().Trim(); string name = n.SelectSingleNode("div/div[2]/div/h4").InnerText.Trim(); name = HttpUtility.HtmlDecode(name); string gia = n.SelectSingleNode("div/div[2]/div/div/span").InnerText.Trim(); string urlSanPham = n.SelectSingleNode("div/div[1]/span/a").Attributes["href"].Value.ToString(); if (string.IsNullOrEmpty(urlImage) || string.IsNullOrEmpty(urlImage) || string.IsNullOrEmpty(urlImage) || string.IsNullOrEmpty(urlImage)) { break; } //HtmlAgilityPack.HtmlDocument htmlDocSanpham = htmlWeb.Load("https://www.adayroi.com" + urlSanPham); //string _htmlSP = htmlDocSanpham.DocumentNode.InnerHtml; //string _ItemInFoBlock = htmlDocSanpham.DocumentNode.SelectSingleNode(@"//div[@id='product_excerpt']").InnerText; //table table-bordered //string _Product_description = htmlDocSanpham.DocumentNode.SelectSingleNode(@"//div[@id='product_description']").InnerHtml; ketqua += brand_id + " $ " + category_id + " $ " + product_item_id + " $ " + merchant_id + " $ " + name + " $ " + gia + " $ " + urlImage + "\n"; } using (FileStream fs = new FileStream(@"F:\DATA.txt", FileMode.Append, FileAccess.Write)) using (StreamWriter w = new StreamWriter(fs, Encoding.UTF8)) { ketqua = HttpUtility.HtmlDecode(ketqua); w.WriteLine(ketqua + "\n"); Console.WriteLine("Trang " + i + " OK"); } } } catch (Exception ex) { Console.WriteLine(ex.Message); Console.ReadLine(); } }
/// <summary> /// Search Shelfari page for possible series info, returning the next title in the series without downloading any other pages. /// TODO: Un-yuckify all the return paths without nesting a ton of ifs /// </summary> /// <param name="searchHtmlDoc">Book's Shelfari page, pre-downloaded</param> private string GetNextInSeriesTitle(HtmlAgilityPack.HtmlDocument searchHtmlDoc) { //Added estimated reading time and page count from Shelfari, for now... HtmlAgilityPack.HtmlNode pageNode = searchHtmlDoc.DocumentNode.SelectSingleNode("//div[@id='WikiModule_FirstEdition']"); if (pageNode == null) { return(""); } HtmlAgilityPack.HtmlNode node1 = pageNode.SelectSingleNode(".//div/div"); if (node1 == null) { return(""); } //Parse page count and multiply by average reading time Match match1 = Regex.Match(node1.InnerText, @"Page Count: ((\d+)|(\d+,\d+))"); if (match1.Success) { double minutes = int.Parse(match1.Groups[1].Value, NumberStyles.AllowThousands) * 1.2890625; TimeSpan span = TimeSpan.FromMinutes(minutes); main.Log(String.Format("Typical time to read: {0} hours and {1} minutes ({2} pages)" , span.Hours, span.Minutes, match1.Groups[1].Value)); curBook.pagesInBook = match1.Groups[1].Value; curBook.readingHours = span.Hours.ToString(); curBook.readingMinutes = span.Minutes.ToString(); } //Added highlighted passage from Shelfari, dummy info for now... HtmlAgilityPack.HtmlNode members = searchHtmlDoc.DocumentNode.SelectSingleNode("//ul[@class='tabs_n tn1']"); int highlights = 0; if (members != null) { Match match3 = Regex.Match(members.InnerText, @"Reviews \(((\d+)|(\d+,\d+))\)"); if (match3.Success) { curBook.popularPassages = match3.Groups[1].Value.ToString(); } match3 = Regex.Match(members.InnerText, @"Readers \(((\d+)|(\d+,\d+))\)"); if (match3.Success) { curBook.popularHighlights = match3.Groups[1].Value.ToString(); highlights = int.Parse(match3.Groups[1].Value, NumberStyles.AllowThousands); } string textPassages = curBook.popularPassages == "1" ? String.Format("{0} passage has ", curBook.popularPassages) : String.Format("{0} passages have ", curBook.popularPassages); string textHighlights = curBook.popularHighlights == "1" ? String.Format("{0} time", curBook.popularHighlights) : String.Format("{0} times", curBook.popularHighlights); main.Log(String.Format("Popular Highlights: {0}been highlighted {1}" , textPassages, textHighlights)); } //If no "highlighted passages" found from Shelfari, add to log if (highlights == 0) { main.Log("Popular Highlights: No highlighted passages have been found for this book"); curBook.popularPassages = ""; curBook.popularHighlights = ""; } //Check if book series is available and displayed in Series & Lists on Shelfari page. HtmlAgilityPack.HtmlNode seriesNode = searchHtmlDoc.DocumentNode.SelectSingleNode("//div[@id='WikiModule_Series']/div"); if (seriesNode != null) { //If multiple Series found, find and use standard series. foreach (HtmlAgilityPack.HtmlNode seriesType in seriesNode.SelectNodes(".//div")) { if (seriesType.InnerText.Contains("(standard series)", StringComparison.OrdinalIgnoreCase) && !seriesType.InnerText.Contains("(Reading Order)", StringComparison.OrdinalIgnoreCase)) { Match match = Regex.Match(seriesType.InnerText, @"This is book (\d+) of (\d+)"); if (!match.Success) { continue; } curBook.seriesName = seriesType.ChildNodes["a"].InnerText.Trim(); main.Log("About the series: " + seriesType.InnerText.Replace(". (standard series)", "")); if (!match.Success || match.Groups.Count != 3) { return(""); } curBook.seriesPosition = match.Groups[1].Value; curBook.totalInSeries = match.Groups[2].Value; HtmlAgilityPack.HtmlNode seriesInfo = seriesNode.SelectSingleNode(".//p"); //Parse preceding book if (seriesInfo != null && seriesInfo.InnerText.Contains("Preceded by ", StringComparison.OrdinalIgnoreCase)) { match = Regex.Match(seriesInfo.InnerText, @"Preceded by (.*),", RegexOptions.IgnoreCase); if (match.Success && match.Groups.Count == 2) { previousTitle = match.Groups[1].Value; } else { match = Regex.Match(seriesInfo.InnerText, @"Preceded by (.*)\.", RegexOptions.IgnoreCase); if (match.Success && match.Groups.Count == 2) { previousTitle = match.Groups[1].Value; } } main.Log("Preceded by: " + previousTitle); //Grab Shelfari Kindle edition link for this book previousShelfariUrl = seriesInfo.ChildNodes["a"].GetAttributeValue("href", "") + "/editions?binding=Kindle"; } // Check if book is the last in the series if (!curBook.seriesPosition.Equals(curBook.totalInSeries)) { //Parse following book if (seriesInfo != null && seriesInfo.InnerText.Contains("followed by ", StringComparison.OrdinalIgnoreCase)) { match = Regex.Match(seriesInfo.InnerText, @"followed by (.*)\.", RegexOptions.IgnoreCase); if (match.Success && match.Groups.Count == 2) { main.Log("Followed by: " + match.Groups[1].Value); //Grab Shelfari Kindle edition link for this book nextShelfariUrl = seriesInfo.ChildNodes["a"].GetAttributeValue("href", "") + "/editions?binding=Kindle"; return(match.Groups[1].Value); } } } //Stop after first standard series is found maybe //add popup (already started implimentaton) in //future to pick which standard series you //want to use, not sure if worthwhile though. //eg: http://www.shelfari.com/books/37598923 break; } } } return(""); }
private object Extract(string name, ConfigSection config, HtmlAgilityPack.HtmlNode parentNode, List <HtmlAgilityPack.HtmlNode> logicalParents) { this.RemoveUnwantedTags(config, parentNode); // We will try to extract text for this item because it does not have children var containers = new JArray(); if (config.XPathRules != null && config.XPathRules.Count > 0) { foreach (var xpath in config.XPathRules) { // TODO: Add try catch Exception var nodes = parentNode.SelectNodes(xpath); if (nodes != null && nodes.Count > 0) { var newLogicalParents = logicalParents.GetRange(0, logicalParents.Count); newLogicalParents.Add(parentNode); foreach (var node in nodes) { if (config.Children != null && config.Children.Count > 0) { var container = new JObject(); this.ExtractChildren(config: config, parentNode: node, container: container, logicalParents: newLogicalParents); containers.Add(container); } else if (config.Transformations != null && config.Transformations.Count > 0) { var obj = this.RunTransformations(config.Transformations, node, newLogicalParents); if (obj != null) { containers.Add(obj); } } else if (node.InnerText != null) { containers.Add(HtmlEntity.DeEntitize(node.InnerText).Trim()); } } } } } else { var container = new JObject(); this.ExtractChildren(config: config, parentNode: parentNode, container: container, logicalParents: logicalParents); containers.Add(container); } if (!config.ForceArray && containers.Count == 0) { return(new JObject()); } else if (!config.ForceArray && containers.Count == 1) { return(containers.First); } else { return(containers); } }
private void textBox1_KeyDown(object sender, KeyEventArgs e) { if (e.KeyCode == Keys.Enter) { String serial = "INVALID_SERIAL"; if (textBox1.Text.Length > 0) { serial = textBox1.Text; } try { webBrowser1.AllowNavigation = true; string parsedHTML = ""; string movieTitle = "UNKNOWN"; string imgSrc = "UNKNOWN"; //string html = new WebClient().DownloadString("https://www.google.com/search?q=" + serial); //parsedHTML = parseHTML(html); //richTextBox1.Text = html; //webBrowser1.DocumentText = html; HtmlWeb web = new HtmlWeb(); document = web.Load("https://www.google.com/search?q=" + serial); HtmlAgilityPack.HtmlNode bodyNode = document.DocumentNode.SelectSingleNode("//td[@id='rhs_block']"); richTextBox1.Text = document.DocumentNode.InnerHtml; if (bodyNode.InnerText.Length > 1) { webBrowser1.DocumentText = bodyNode.InnerHtml; foreach (var image in bodyNode.SelectNodes(".//img")) { var src = image.GetAttributeValue("src", null); if (src != null) { imgSrc = src; break; } } textBox2.Enabled = false; normalQuery = true; try { movieTitle = bodyNode.SelectSingleNode("//div[@class='_B5d']").InnerText; } catch (Exception ex) { textBox2.Enabled = true; } } else { normalQuery = false; textBox2.Enabled = true; webBrowser1.Navigate("https://www.google.com/search?q=" + serial);// = new WebClient().DownloadString("https://www.google.com/search?q=" + serial); } if (list.Count == 0) { list.Add(new String[] { serial + " (1)", movieTitle, imgSrc }); listView1.Items.Add(serial + " (1)"); listBox2.Items.Add(movieTitle); } else { for (int i = 0; i < list.Count; ++i) { if (list[i][0].ToString().Contains(serial)) { int count = Int32.Parse(list[i][0].ToString().Substring(list[i][0].ToString().IndexOf('(')).Substring(1, list[i][0].ToString().Substring(list[i][0].ToString().IndexOf('(')).Length - 2)); list[i][0] = list[i][0].ToString().Substring(0, list[i][0].ToString().IndexOf('(')) + "(" + ++count + ")"; listView1.Items.Clear(); listBox2.Items.Clear(); foreach (String[] j in list) { listView1.Items.Add(j[0]); listBox2.Items.Add(j[1]); } i = list.Count; break; } else if (i == list.Count - 1) { list.Add(new String[] { serial + " (1)", movieTitle, imgSrc }); listView1.Items.Add(serial + " (1)"); listBox2.Items.Add(movieTitle); i = list.Count; break; } } } } } catch (Exception ex) { textBox1.Text = "INVALID SERIAL"; } textBox1.Text = ""; e.SuppressKeyPress = true; } }
public override void Parse(Response response) { //Create a new HTMLAglityPack document HtmlDocument ContentDocument = new HtmlDocument(); //load the #content of the page into the document ContentDocument.LoadHtml(response.Css("#content").First().OuterHtml); HtmlAgilityPack.HtmlNode BodyNode = ContentDocument.DocumentNode; patternObject.Title = BodyNode.SelectSingleNode("//*[@id=\"firstHeading\"]").InnerHtml; HtmlAgilityPack.HtmlNode ContentNode = BodyNode.SelectSingleNode("//*[@id=\"mw-content-text\"]"); //remove the "toc" and "jump" and "siteSub" sections to save space and later client-side processing time if (ContentNode.SelectSingleNode("//*[@id=\"toc\"]") != null) { ContentNode.SelectSingleNode("//*[@id=\"toc\"]").Remove(); } foreach (var node in ContentNode.SelectNodes("//comment()")) { node.Remove(); } ContentNode.PrependChild(BodyNode.SelectSingleNode("//*[@id=\"firstHeading\"]")); //set the patternObject's title patternObject.Title = ContentNode.SelectSingleNode("//*[@id=\"firstHeading\"]").InnerHtml; foreach (var link in ContentNode.SelectNodes("//a/@href")) { //skip if this is a redlink (page doesn't exist). if (link.Attributes["href"].Value.Contains("redlink=1")) { continue; } //skip if this links to this page if (link.Attributes["href"].Value.Split('#').First() == response.FinalUrl) { continue; } //if any of the links ancestor nodes is the "category links" part of the page if (link.Ancestors().Any(node => node.Id == "catlinks")) { if (link.InnerText != "Categories") //if it is not the "categories" special page { //add it to the patterns list of categories patternObject.Categories.Add(link.InnerText); } } else //assume its a normal text-body link { //check if we don't already know about this link patternObject.CreateOrGetPatternLink(link.InnerText); } //add relation info if this is a relation link if (GetNodeReleventPageHeading(link, "h2") != null && GetNodeReleventPageHeading(link, "h2").InnerText == "Relations") { //get the relation type of this relation and get its inner text HtmlAgilityPack.HtmlNode RelationHeadingNode = GetNodeReleventPageHeading(link, "h3"); String RelationName = RelationHeadingNode.InnerText; //if there is a h4 node before the previous h3 node if (GetNodeReleventPageHeading(link, "h4") != null && RelationHeadingNode.InnerStartIndex < GetNodeReleventPageHeading(link, "h4").InnerStartIndex) { //assume it is a "with x" sub-category of relation for the "Can Instantiate" section RelationName = RelationHeadingNode.InnerText + " " + GetNodeReleventPageHeading(link, "h4").InnerText; } //add the relevent relation to this link patternObject.CreateOrGetPatternLink(link.InnerText).Type.Add(RelationName); } } //get a cleaned copy of the #content HTML for giving in the JSON data patternObject.Content = ProcessPageContentToString(ContentNode); string Json = JsonConvert.SerializeObject(patternObject); File.WriteAllText(Pattern.GetFileName(patternObject.Title), Json); }
public static IList <HtmlNode> SelectNodesAsList( this HtmlNode node, string xpath) => node.SelectNodes(xpath)?.ToList() ?? new List <HtmlNode>(0);
public List <Entry> parseFile(string file) { List <Entry> result = new List <Entry>(); if (!File.Exists(file)) { log.Error(String.Format("File \"{0}\" does not exist.", file)); } else { log.InfoFormat("Parsing file : {0}", file); HtmlDocument htmlDoc = new HtmlDocument(); htmlDoc.Load(file); if (htmlDoc.DocumentNode != null) { HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body"); foreach (HtmlParseError error in htmlDoc.ParseErrors) { log.Error(String.Format("Error \nCode: {0}\nLine: {1}\nPosition : {2}\nReason : {3}\nSource Text : {4} ", error.Code, error.Line, error.LinePosition, error.Reason, error.SourceText)); } if (htmlDoc.ParseErrors.Count() == 0) { if (bodyNode != null) { int year = extractYear(bodyNode); HtmlNodeCollection monthsNodeCollection = bodyNode.SelectNodes("//h3"); foreach (HtmlNode monthNode in monthsNodeCollection) { Month month = extractMonth(monthNode); HtmlNodeCollection entryNodeCollection = monthNode.SelectNodes("following-sibling::div[1]/ul/li/h2"); // If entries exist for the month, resume ... if (entryNodeCollection != null) { foreach (HtmlNode entryNode in entryNodeCollection) { DateTime entryDate = extractEntryDate(entryNode); String entryTitle = extractEntryTitle(entryNode); DateStamp entryKey = new DateStamp() { Year = year, Month = month, Day = entryDate.Day }; String entryText = extractEntryText(entryNode); List <Image> entryImages = extractEntryImages(entryNode); result.Add( new Entry() { Key = entryKey, Title = entryTitle, Text = entryText, Images = entryImages }); } } else { log.Error(String.Format("Could not find any entries for month. Html: {0}", monthNode.InnerHtml)); } } } } } } return(result); }
public void Crawler() { int previousPageGallNum = 1000000000; Console.WriteLine(initDate.ToString() + endDate.ToString()); string url = gallUrl + "&page="; var client = new WebClient(); client.Encoding = System.Text.Encoding.UTF8; //Dictionary value => count, replyNum, gallCount, gallRecommend Dictionary <UserInfo, int[]> userDic = new Dictionary <UserInfo, int[]>(); int currentPage = this.initPage; while (true) { string text; try { text = client.DownloadString(url + currentPage.ToString()); if (string.IsNullOrEmpty(text)) { continue; } } catch { continue; } hap.HtmlDocument textHap = new hap.HtmlDocument(); textHap.LoadHtml(text); hap.HtmlNodeCollection nicks = textHap.DocumentNode.SelectNodes("//tr[@class='ub-content us-post']"); //Console.WriteLine(nicks.Count); //Console.WriteLine("==================" + currentPage.ToString() + "=================="); try { foreach (hap.HtmlNode nick in nicks) { int gallNum, replyNum, gallCount, gallRecommend; DateTime gallDate; string subject; gallNum = GetOnlyInt(nick.SelectSingleNode("./td[@class='gall_num']").InnerText); gallDate = DateTime.ParseExact(nick.SelectSingleNode("./td[@class='gall_date']").Attributes["title"].Value, "yyyy-MM-dd HH:mm:ss", null); Console.WriteLine(gallNum.ToString() + " " + gallDate.ToString()); if (gallNum >= previousPageGallNum) { Console.WriteLine(previousPageGallNum.ToString() + " " + gallNum.ToString()); Console.WriteLine("번호 에러"); continue; } if (DateTime.Compare(gallDate, initDate) < 0 || DateTime.Compare(gallDate, endDate) > 0) { Console.WriteLine("날짜 에러"); continue; } hap.HtmlNode user = nick.SelectSingleNode("./td[@class='gall_writer ub-writer']"); UserInfo tempUserInfo = new UserInfo(user.Attributes["data-nick"].Value); if (user.Attributes["data-uid"].Value == "") { tempUserInfo.setFluidNick(user.Attributes["data-ip"].Value); } else { tempUserInfo.setFixedNick(user.Attributes["data-uid"].Value); } //replyNum and subject are in <td class='gall_tit ub-word'></td> hap.HtmlNode subjectNode = nick.SelectSingleNode("./td[2]"); try { if (subjectNode.Attributes["class"].Value == "gall_subject") { subjectNode = nick.SelectSingleNode("./td[3]"); } subject = subjectNode.SelectSingleNode("./a[1]").InnerText; if (subjectNode.SelectNodes("./a").Count == 2) { replyNum = GetOnlyInt(subjectNode.SelectSingleNode("./a[@class='reply_numbox']/span").InnerText); } else { replyNum = 0; } } catch { subject = "NullSubjectException"; replyNum = 0; } // Console.WriteLine("댓글: " + replyNum.ToString()); gallCount = GetOnlyInt(nick.SelectSingleNode("./td[@class='gall_count']").InnerText); gallRecommend = GetOnlyInt(nick.SelectSingleNode("./td[@class='gall_recommend']").InnerText); //Dictionary value => count, replyNum, gallCount, gallRecommend if (userDic.ContainsKey(tempUserInfo)) { userDic[tempUserInfo][0] += 1; userDic[tempUserInfo][1] += replyNum; userDic[tempUserInfo][2] += gallCount; userDic[tempUserInfo][3] += gallRecommend; } else { int[] tempInts = new int[] { 1, replyNum, gallCount, gallRecommend }; userDic.Add(tempUserInfo, tempInts); } UserData tempUserData = new UserData(tempUserInfo); tempUserData.DataInput(gallNum, replyNum, gallCount, gallRecommend, gallDate, subject); //gallDatas.Add(tempUserData); } } catch { if (ErrorOccured != null) { ErrorOccured(text, null); } currentPage++; continue; } previousPageGallNum = GetOnlyInt(nicks[nicks.Count - 1].SelectSingleNode("./td[@class='gall_num']").InnerText); DateTime currentDate = DateTime.ParseExact(nicks[nicks.Count - 1]. SelectSingleNode("./td[@class='gall_date']").Attributes["title"].Value, "yyyy-MM-dd HH:mm:ss", null); if (currentPage >= endPage || DateTime.Compare(currentDate, initDate) < 0) { break; } else { System.Collections.ArrayList arr = new System.Collections.ArrayList(); string str = currentPage.ToString() + " 페이지, 날짜: " + currentDate.ToString(); arr.Add(str); arr.Add(currentDate); arr.Add(currentPage - initPage); if (newPageHappened != null) { newPageHappened(arr, null); } currentPage++; } } //Dictionary value => count, replyNum, gallCount, gallRecommend foreach (KeyValuePair <UserInfo, int[]> user in userDic) { UserInfo tempUser = user.Key; tempUser.count = user.Value[0]; tempUser.replyNum = user.Value[1]; tempUser.gallCount = user.Value[2]; tempUser.gallRecommend = user.Value[3]; UserRank tempUserRank = new UserRank(tempUser, user.Value[0], user.Value[1], user.Value[2], user.Value[3]); userList.Add(tempUserRank); } var sorted = from userRank in userList orderby userRank.count descending select userRank; userList = sorted.ToList <UserRank>(); if (CrawlingEnded != null) { CrawlingEnded(userList, null); } string tempDataDir = Directory.GetCurrentDirectory() + "\\temp-data\\"; Directory.CreateDirectory(tempDataDir); string filename = tempDataDir + gallId + DateTime.Now.ToString("_yyyy-MM-dd_HH-mm-ss"); SaveResult(filename); }
/// <summary> /// return node collection that match _LabelData.XPath /// </summary> /// <returns></returns> protected HtmlAgilityPack.HtmlNodeCollection GetNodeCollection() { HtmlAgilityPack.HtmlNode ndTempParent = GetTempParentNode(); return(ndTempParent.SelectNodes(_LabelData.XPath)); }