private void parseBTN_Click(object sender, RoutedEventArgs e) { for (int j = 1; j <= Int32.Parse(countTB.Text); j++) { string url = linkTB.Text + "?page=" + j; var web = new HtmlWeb(); var doc = web.LoadFromBrowser(url); var names = doc.DocumentNode.SelectNodes("//li[@class = 'catalog__item inner ng-scope']//p[@class = 'catalog__name']"); var prices = doc.DocumentNode.SelectNodes("//li[@class = 'catalog__item inner ng-scope']//p[@class = 'catalog__price ng-scope ng-binding']"); for (int i = 0; i <= names.Count - 1; i++) { string priceNumbers = RemoveNonNumeric(prices[i].InnerText); Plywood good = new Plywood { GoodName = names[i].InnerText.Trim(), GoodPrice = Double.Parse(priceNumbers.Remove(priceNumbers.Length - 2)), Height = ExtractDimensions(names[i].InnerText.Trim()).Split('-')[0], Width = ExtractDimensions(names[i].InnerText.Trim()).Split('-')[1], Depth = ExtractDimensions(names[i].InnerText.Trim()).Split('-')[2] }; good.Square = Double.Parse(good.Height) * Double.Parse(good.Width) / 1000000; good.GoodPriceM2 = good.GoodPrice / good.Square; goodsDGRID.Items.Add(good); } } }
static void Main(string[] args) { string url = "http://html-agility-pack/from-browser"; var web1 = new HtmlWeb(); var doc1 = web1.LoadFromBrowser(url, o => { var webBrowser = (WebBrowser)o; // WAIT until the dynamic text is set return(!string.IsNullOrEmpty(webBrowser.Document.GetElementById("uiDynamicText").InnerText)); }); var t1 = doc1.DocumentNode.SelectSingleNode("//div[@id='uiDynamicText']").InnerText; var web2 = new HtmlWeb(); var doc2 = web2.LoadFromBrowser(url, html => { // WAIT until the dynamic text is set return(!html.Contains("<div id=\"uiDynamicText\"></div>")); }); var t2 = doc2.DocumentNode.SelectSingleNode("//div[@id='uiDynamicText']").InnerText; Console.WriteLine("Text 1: " + t1); Console.WriteLine("Text 2: " + t2); }
public List <WebData> GetShopHiveDataAsync(string productName) { List <WebData> shophive = new List <WebData>(); try { var web2 = new HtmlWeb(); web2.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36"; web2.BrowserTimeout = TimeSpan.Zero; var doc2 = web2.LoadFromBrowser(string.Format("http://www.shophive.com/catalogsearch/result/?cat=0&q={0}", productName), html => { // WAIT until the dynamic text is set return(!html.Contains("<div class=\"products-grid\"></div>")); }); var t1 = doc2.DocumentNode.SelectNodes(".//div[@class='product-block']"); foreach (HtmlNode ht in t1) { shophive.Add(new WebData { Name = ht.SelectNodes(".//div[@class='product-block-inner']/a")[0].Attributes["title"].Value, Link = ht.SelectNodes(".//div[@class='product-block-inner']/a")[0].Attributes["href"].Value, Image = ht.SelectNodes(".//div[@class='product-block-inner']/a/img")[0].Attributes["src"].Value, Price = ht.SelectNodes(".//div[@class='price-box']/span")[0].InnerText }); } } catch (Exception ex) { shophive = null; } return(shophive); }
public static HtmlDocument LoadPagefromBrower(string url) { HtmlWeb webAgent = new HtmlWeb(); HtmlDocument htmlDocument = webAgent.LoadFromBrowser(url); return(htmlDocument); }
public List <WebData> GetSymbiosDataAsync(string productName) { List <WebData> symbios = new List <WebData>(); try { var web2 = new HtmlWeb(); web2.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36"; web2.BrowserTimeout = TimeSpan.Zero; var doc2 = web2.LoadFromBrowser(string.Format("https://www.symbios.pk/search.php?search_query={0}", productName), html => { // WAIT until the dynamic text is set return(!html.Contains("<div id=\"category-products\"></div>")); }); int i = 0; foreach (HtmlNode li in doc2.DocumentNode.SelectNodes("//div[contains(@class,'products')]")) { symbios.Add(new WebData() { Link = li.SelectNodes(@"//div[@class='product-info text-center']/h3[@class='name']/a")[i].Attributes["href"].Value, Price = li.SelectNodes(@"//div[@class='product-info text-center']/div[@class='product-price']/span[@class='price']")[i].InnerText, Image = li.SelectNodes(@"//div[@class='product-image']/a/img")[i].Attributes["src"].Value, Name = li.SelectNodes(@"//div[@class='product-info text-center']/h3[@class='name']/a")[i].InnerText, }); i++; } } catch (Exception ex) { symbios = null; } return(symbios); }
static void Main(string[] args) { const string SECTION_CLASS = "WyeMbd"; const string SECTION_HEADER_CLASS = "adH5zf"; const string MAIN_COVERAGE_CLASS = "nuEeue"; var csv = new StringBuilder(); string filePath = "headlines.csv"; var URL = @"https://news.google.com/"; HtmlWeb web = new HtmlWeb(); var htmlDoc = web.LoadFromBrowser(URL); var frontpageNewsSections = htmlDoc.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains(SECTION_CLASS)); List <string> headers = new List <string>(); List <List <string> > allMainHeadlines = new List <List <string> >(); foreach (var section in frontpageNewsSections) { var header = section.Descendants().Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains(SECTION_HEADER_CLASS)).First().InnerText; var mainHeadlines = section.Descendants().Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains(MAIN_COVERAGE_CLASS)); headers.Add(header); Console.WriteLine(header); allMainHeadlines.Add(new List <string>()); foreach (var headline in mainHeadlines) { var santizedHeadline = WebUtility.HtmlDecode(headline.InnerText); allMainHeadlines.Last().Add(santizedHeadline); Console.WriteLine("\t" + santizedHeadline); } } foreach (var header in headers) { csv.Append(header + ","); } csv.Append(Environment.NewLine); int maxHeadLines = allMainHeadlines.Select(x => x.Count).ToList().Max(); for (int i = 0; i < maxHeadLines; i++) { foreach (var sectionHeadlines in allMainHeadlines) { if (i < sectionHeadlines.Count) { csv.Append('"' + sectionHeadlines[i] + '"' + ","); } } csv.Append(Environment.NewLine); } File.WriteAllText(filePath, csv.ToString()); Console.WriteLine(); Console.WriteLine("Output exported to " + filePath); Console.ReadKey(); }
public List <WebData> GetYayvoDataAsync(string productName) { List <WebData> yayvo = new List <WebData>(); try { var web2 = new HtmlWeb(); web2.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36"; web2.BrowserTimeout = TimeSpan.Zero; var doc2 = web2.LoadFromBrowser(string.Format("http://yayvo.com/search/result/?q={0}", productName), html => { // WAIT until the dynamic text is set return(!html.Contains("<ul class=\"products-grid\"></ul>")); }); var liList = doc2.DocumentNode.SelectNodes("//ul[contains(@class, 'products-grid')]"); foreach (HtmlNode li in liList) { yayvo.Add(new WebData() { Link = li.Descendants("a").First().Attributes["href"].Value, Name = li.Descendants("a").First().Attributes["title"].Value, Image = li.Descendants("a").First().Descendants("img").First().Attributes["src"].Value, Price = li.SelectSingleNode(@"//div[@class='price-box']/p[@class='special-price']/span").InnerText }); } } catch (Exception ex) { yayvo = null; } return(yayvo); }
private void button1_Click(object sender, EventArgs e) { HtmlWeb hw = new HtmlWeb(); HtmlAgilityPack.HtmlDocument doc = hw.LoadFromBrowser("Insert URL"); // foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a")){ Debug.WriteLine(link.Attributes["href"].Value); } // string[] Links = new string[3015]; // The Total var ssa = 0; foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]")) { //Debug.WriteLine(link.InnerText); if (link.InnerText.Contains(".jpg") || link.InnerText.Contains(".png") || link.InnerText.Contains(".gif") || link.InnerText.Contains(".jpeg")) { treeView1.Nodes.Add(link.Attributes["href"].Value); // Links[ssa] = link.InnerText; } ssa++; Debug.WriteLine(link.Id.ToString()); //if (link.Name) { } // Debug.WriteLine(link.); } Debug.WriteLine(ssa); Debug.WriteLine(treeView1.Nodes.Count); }
private List <String> downloadAndProcessKaduEpisodes(String url) { HtmlWeb web = new HtmlWeb(); var htmlDoc = web.LoadFromBrowser(url); var nodes = htmlDoc.DocumentNode.SelectNodes("//div[@id='video-line-content']/a"); return(nodes.Select(x => "http://kadu.ru" + x.Attributes["href"].Value).ToList()); }
private HtmlDocument GetHtmlDocument(string url, TimeSpan outTime, Func <string, bool> isScriptCompleted = null) { try { var htmlWeb = new HtmlWeb() { BrowserTimeout = outTime, UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36", CaptureRedirect = true }; return(isScriptCompleted == null? htmlWeb.LoadFromBrowser(url) : htmlWeb.LoadFromBrowser(url, isScriptCompleted)); } catch (Exception ex) { throw ex; } }
private HtmlBelge Indir() { HtmlWeb web = new HtmlWeb { BrowserTimeout = new TimeSpan(0) }; HtmlBelge htmlBelgeYukle = web.LoadFromBrowser(SecilenSunucuAdresi()); return(htmlBelgeYukle); }
public string GetLyric(string songName, string artist = null) { if (InternetConnection.IsConnectedToInternet() == false) { return(string.Empty); } if (string.IsNullOrWhiteSpace(artist)) { document = html.LoadFromBrowser("https://mp3.zing.vn/tim-kiem/bai-hat.html?q=" + songName); } else { document = html.LoadFromBrowser("https://mp3.zing.vn/tim-kiem/bai-hat.html?q=" + songName + ' ' + artist); } string link = null; try { var data = document.DocumentNode.QuerySelector(".fl .title-song h3 a"); link = @"https://mp3.zing.vn" + data.Attributes["href"].DeEntitizeValue; document = html.LoadFromBrowser(link); var lyric = document.DocumentNode.QuerySelector(".fn-container [id] p"); document = null; if (lyric == null) { return(string.Empty); } else { return(lyric.InnerHtml.Replace("<br>", "\n")); } } catch { document = null; return(string.Empty); } finally { GC.Collect(); } }
public void GetAllHrefs(String uri) { var web = new HtmlWeb(); web.BrowserTimeout = TimeSpan.FromSeconds(30); var doc = web.LoadFromBrowser(uri, o => { // var webBrowser = (WebBrowser)o; return(o.Contains("card-content-title")); }); GC.Collect(); // Show results Regex re = new Regex(@">(.*?)<"); foreach (var title in doc.DocumentNode.SelectNodes(".//div[@class='card-content']")) { if (title.SelectSingleNode("a[@class='card-content-title']") != null) { string url = "https://docs.microsoft.com/en-us" + title.SelectSingleNode("a[@class='card-content-title']").GetAttributeValue("href", String.Empty); hrefslist.Add(url); if (title.SelectSingleNode("ul[@class='tags']") != null) { string result = ""; string a = (title.SelectSingleNode("ul[@class='tags']").InnerText); foreach (Match match in re.Matches(a)) { if (match.Groups[1].Length != 0) { result = match.Groups[1].Value; break; } /* if(!match.Groups[1].Equals("Advanced") && !match.Groups[1].Equals("Intermediate") && !match.Groups[1].Equals("Beginner") ) * { * result += match.Groups[1] + ", "; * } */ } // if(result.Length>2) result = result.Substring(0, result.Length - 2); prodDictionary[url] = result; } } } }
public static void PopulateCategory(Mouser.Domain.ApplicationContext context) { try { string currentUri = "https://www2.mouser.com/Electronic-Components/"; var web = new HtmlWeb(); var doc = web.LoadFromBrowser(HttpUtility.HtmlDecode(currentUri), o => { WebBrowser webBrowser = (WebBrowser)o; // WAIT until the dynamic text is set return(!string.IsNullOrEmpty(webBrowser.Document.GetElementById("mlnkMailTo").InnerText)); }); var webCategories = new List <WebCategory>(); var categoryNodes = doc.DocumentNode.SelectNodes("//*[@class=\"panel light-grey-panel\"]"); foreach (var node in categoryNodes) { var categoryName = node.SelectSingleNode(".//h2[1]").InnerText; var subCategoryNameNodes = node.SelectNodes(".//a[@class=\"SearchResultsSubLevelCategory\"]"); foreach (var subCategoryName in subCategoryNameNodes) { webCategories.Add(new WebCategory { Category = categoryName.Replace("&", "&"), SubCategory = subCategoryName.InnerText.Substring(0, subCategoryName.InnerText.LastIndexOf(" ") - 1).Replace("&", "&") }); } } foreach (var wc in webCategories) { List <Category> categories = context.Categories.Include(c => c.Manufacturer).Where(c => !c.IsCategory && c.Name == wc.SubCategory).ToList(); foreach (Category category in categories) { var cat = context.Categories.FirstOrDefault(c => c.IsCategory && c.Name == wc.Category && c.Manufacturer.Id == category.Manufacturer.Id); if (cat == null) { cat = new Category { IsCategory = true, Manufacturer = category.Manufacturer, Name = wc.Category }; context.Categories.Add(cat); context.SaveChanges(); } category.ParentId = cat.Id; context.SaveChanges(); } } } finally { } }
static void Main(string[] args) { // 348050423: 剑如长虹气如玉 _activeUserId = ConfigurationManager.AppSettings["UserId"]; if (_activeUserId == null) { throw new Exception("AppSettings未配置UserId"); } var firstPageUrl = GetVideoUrl(_activeUserId, 1); _web = new HtmlWeb(); var document = _web.LoadFromBrowser(firstPageUrl, o => { var webBrowser = (WebBrowser)o; // WAIT until the dynamic text is set return(webBrowser.Document.GetElementById("submit-video-list") != null && webBrowser.Document.GetElementById("submit-video-list").Children[1].Children.Count > 0); }); var pageNumNode = document.DocumentNode.SelectSingleNode(".//div[@id='submit-video-list']/ul[@class='be-pager']/span[@class='be-pager-total']"); // 总页数 var pageCount = GetPageCount(pageNumNode); var totalCountNode = document.DocumentNode.SelectSingleNode(".//div[@id='submit-video-type-filter']/a[1]/span"); var totalCount = totalCountNode.InnerText; Console.WriteLine($"总页数: { pageCount}, 总个数: {totalCount}"); if (pageCount <= 0) { return; } StringBuilder result = new StringBuilder(500); result.AppendLine($"总数: {totalCount}, 生成时间{DateTime.Now}"); result.AppendLine(HEADER); for (int i = 1; i <= pageCount; i++) { Console.WriteLine($"第{i}页读取中..."); var videoInfo = GetVideosFromNode(i); result.Append(videoInfo); } System.IO.File.WriteAllText($"export-{_activeUserId}.csv", result.ToString()); Console.WriteLine("Load Complete"); }
private void simpleButton6_Click(object sender, EventArgs e) { var web2 = new HtmlWeb(); string currentUri = @"https://ru.mouser.com/ProductDetail/TE-Connectivity/M85049-93-16?qs=2WXlatMagcH45qgLJ7jojA%3D%3D"; var doc2 = web2.LoadFromBrowser(HttpUtility.HtmlDecode(currentUri), o => { WebBrowser webBrowser = (WebBrowser)o; // WAIT until the dynamic text is set return(!string.IsNullOrEmpty(webBrowser.Document.GetElementById("mlnkMailTo").InnerText)); }); int sds = 3; return; try { Console.WriteLine(String.Format("*** Begin Request ***")); var request = (System.Net.HttpWebRequest)System.Net.WebRequest.Create("http://api.PhantomJScloud.com/api/browser/v2/a-demo-key-with-low-quota-per-ip-address/"); request.ContentType = "application/json"; request.Method = "POST"; request.Timeout = 45000; //45 seconds request.KeepAlive = false; request.MediaType = "application/json"; request.ServicePoint.Expect100Continue = false; //REQUIRED! or you will get 502 Bad Gateway errors using (var streamWriter = new System.IO.StreamWriter(request.GetRequestStream())) { //you should look at the HTTP Endpoint docs, section about "userRequest" and "pageRequest" //for a listing of all the parameters you can pass via the "pageRequestJson" variable. string pageRequestJson = @"{'url':'https://ru.mouser.com/ProductDetail/TE-Connectivity/M85049-93-16?qs=2WXlatMagcH45qgLJ7jojA%3D%3D','renderType':'plainText','outputAsJson':true }"; streamWriter.Write(pageRequestJson); streamWriter.Flush(); streamWriter.Close(); } var response = (System.Net.HttpWebResponse)request.GetResponse(); Console.WriteLine(String.Format("HttpWebResponse.StatusDescription: {0}", response.StatusDescription)); using (var streamReader = new System.IO.StreamReader(response.GetResponseStream())) { string server_reply = streamReader.ReadToEnd(); Console.WriteLine(String.Format("Server Response content: {0}", server_reply)); } response.Close(); } catch (Exception Ex) { Console.WriteLine("*** HTTP Request Error ***"); Console.WriteLine(Ex.Message); } }
private String downloadKaduVideoToFolder(String url, String saveToPath) { HtmlWeb web = new HtmlWeb(); var htmlDoc = web.LoadFromBrowser(url); var node = htmlDoc.DocumentNode.SelectSingleNode("//video[@id='video-flash']"); var videoUrl = node.Attributes["src"].Value; var titleNode = htmlDoc.DocumentNode.SelectSingleNode("//h1"); var title = titleNode.InnerHtml; int t = title.IndexOf("</span>"); title = title.Substring(t, title.Length - t) + ".mp4"; using (var webClient = new WebClient()) { webClient.DownloadFile(videoUrl, title); } return(title); }
public string GetHtmlDocument(string url) { var webPage = new HtmlWeb(); HtmlDocument pageContents = new HtmlDocument(); try { pageContents = webPage.LoadFromBrowser(url); } catch (Exception e) { Console.WriteLine("Sorry that was an invalid url. Please enter a valid one."); string newUrl = Console.ReadLine(); GetHtmlDocument(newUrl); } return(pageContents.ParsedText); }
public static List <string> GetGoogleLinks(string URl, out bool done) { List <string> rtn = new List <string>(); try { var web = new HtmlWeb(); var xPath = @"//div[contains(concat(' ', normalize-space(@class), ' '), ' r ')]"; var doc = web.LoadFromBrowser(URl); var nodes = doc.DocumentNode.SelectNodes(xPath); foreach (var item in nodes) { var url = item.ChildNodes.FindFirst("a").Attributes["href"].Value; if (url.StartsWith("http")) { try { doc = web.Load(url); var Nodes = GetNodes(doc); if (Nodes != null) { rtn.Add(url); } } catch (Exception) { } } } } catch (Exception ex) { Xceed.Wpf.Toolkit.MessageBox.Show("An error occured. \n Please check your internet connection and try again.\n" + ex.Message); } finally { done = true; } return(rtn); }
/// <summary> /// Returns the Bank NIFTY page URL from the main market URL JavaScript snippet. /// </summary> /// <param name="marketURL">The market URL for Bank NIFTY.</param> /// <returns>The Bank NIFTY URL.</returns> private string GetBankNIFTYMarketURL(string marketURL) { // Load the web page and get the JavaScript var htmlWeb = new HtmlWeb(); HtmlAgilityPack.HtmlDocument htmlDocument = htmlWeb.LoadFromBrowser(marketURL); HtmlNodeCollection scriptTags = htmlDocument.DocumentNode.SelectNodes("//script[@type=\"text/javascript\"]"); string bankNIFTYMarketURL = scriptTags[17].InnerHtml; // Process the JavaScript and get the Bank NIFTY URL bankNIFTYMarketURL = bankNIFTYMarketURL.Replace("\r\n", ""); bankNIFTYMarketURL = bankNIFTYMarketURL.Replace("\t", ""); bankNIFTYMarketURL = bankNIFTYMarketURL.Remove(0, bankNIFTYMarketURL.IndexOf("BANKNIFTY")); bankNIFTYMarketURL = bankNIFTYMarketURL.Remove(0, 28); bankNIFTYMarketURL = bankNIFTYMarketURL.Remove(bankNIFTYMarketURL.IndexOf(";"), bankNIFTYMarketURL.Length - bankNIFTYMarketURL.IndexOf(";")); bankNIFTYMarketURL = bankNIFTYMarketURL.Replace("'", ""); return(NSEIndiaWebsiteURL + "/" + bankNIFTYMarketURL); }
public static void CheckForUpdate() { if (File.Exists("AtlusScriptCompiler.exe")) { string version = ""; string download = ""; // Load webpage in browser HtmlWeb web = new HtmlWeb(); web.BrowserTimeout = TimeSpan.FromSeconds(0); var appveyorPage = web.LoadFromBrowser("https://ci.appveyor.com/project/TGEnigma/atlusscripttools/build/artifacts", o => { var webBrowser = (WebBrowser)o; // Wait until the download url loads return(webBrowser.Document.Body.InnerHtml.Contains("artifact-type")); }); // Scrape version and download url from HTML version = appveyorPage.DocumentNode.SelectSingleNode("//div[@class='project-build-version ng-binding']").InnerText; download = appveyorPage.DocumentNode.SelectSingleNode("//a[@class='artifact-type zip']").Attributes["href"].Value; if (version != "" && download != "") { // Create/Modify Update XML File.WriteAllText("updates.xml", $"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<item>\n\t<version>{version}.0</version>\n\t<url>{download}</url>\n\t<changelog>https://github.com/TGEnigma/Atlus-Script-Tools/commits/master</changelog>\n\t<mandatory>false</mandatory>\n</item>"); //Wait for XML file to exist/not be in use using (WaitForFile("updates.xml", FileMode.Open, FileAccess.ReadWrite, FileShare.None)) { }; // Ask to download and install update from XML data AutoUpdater.RunUpdateAsAdmin = false; AutoUpdater.InstalledVersion = new Version(AssemblyName.GetAssemblyName("AtlusScriptCompiler.exe").Version.ToString()); AutoUpdater.Start("updates.xml"); } } else { MessageBox.Show("Could not find AtlusScriptCompiler.exe. " + "Put this program in the same folder and try running it again!", "Critical Error", MessageBoxButtons.OK, MessageBoxIcon.Exclamation); } }
public TabControl RenderizeSeasonTab(string lastSeasonUrl, SearchSources sourceName) { HtmlAgilityPack.HtmlDocument htmldoc = _web.LoadFromBrowser(lastSeasonUrl); List <ISourceScraperData> scraperData = new List <ISourceScraperData>(); IEnumerable <HtmlNode> episodes; switch (sourceName) { case SearchSources.TuSubtitulo: episodes = htmldoc.DocumentNode.Descendants("table").Skip(1); foreach (HtmlNode episode in episodes) { TuSubtituloScraperData scraperItem = new TuSubtituloScraperData(); scraperItem.SetEpisodeData(episode, "https:"); scraperData.Add(scraperItem); } break; case SearchSources.Subtitulamos: HtmlNode episodesWrapper = htmldoc.DocumentNode.Descendants("div").SingleOrDefault(x => x.Id == "episode-choices"); episodes = episodesWrapper.Descendants("a"); foreach (HtmlNode episode in episodes) { HtmlAgilityPack.HtmlDocument episodeHtml = _web.Load("https://www.subtitulamos.tv" + episode.Attributes["href"].Value); SubtitulamosScraperData scraperItem = new SubtitulamosScraperData(); scraperItem.SetEpisodeData(episodeHtml.DocumentNode.Descendants("div").SingleOrDefault(e => e.Attributes.Contains("class") && e.Attributes["class"].Value.Equals("content limited-width")), "https://www.subtitulamos.tv"); scraperData.Add(scraperItem); } break; default: MessageBox.Show("No se pudo generar los resultados. El servidor de datos para la búsqueda es incorrecta."); break; } _sourceName = sourceName; _TabCtrlResults = SetControls(scraperData, _TabCtrlResults); return(_TabCtrlResults); }
static string GetVideosFromNode(int pageNum) { var videoUrl = GetVideoUrl(_activeUserId, pageNum); //var document = _web.LoadFromBrowser(videoUrl); var document = _web.LoadFromBrowser(videoUrl, o => { var webBrowser = (WebBrowser)o; // WAIT until the dynamic text is set return(webBrowser.Document.GetElementById("submit-video-list") != null && webBrowser.Document.GetElementById("submit-video-list").Children[1].Children.Count > 0); }); // 第一个ul是详细模式, 第二个ul是表格模式 var videoNodes = document.DocumentNode.SelectNodes(".//div[@id='submit-video-list']/ul[2]/li"); if (videoNodes == null) { System.Console.WriteLine("no nodes"); return(string.Empty); } StringBuilder result = new StringBuilder(); foreach (var item in videoNodes) { var titleNode = item.SelectSingleNode(".//a[@class='title']"); var title = titleNode.Attributes["title"].Value; var url = "https:" + titleNode.Attributes["href"].Value; var videoTimeNode = item.SelectSingleNode(".//a[@class='cover']/span[@class='length']"); var videoTime = videoTimeNode.InnerText; var metaNode = item.SelectSingleNode(".//div[@class='meta']/span[@class='time']"); var createTime = metaNode.InnerText.TrimEnd(' ', '\n'); result.AppendLine($"{title}\t{videoTime}\t{url}\t{createTime}\t{pageNum}"); } return(result.ToString()); }
public static ServerTinfo GetSSbyArea(string area) { //view-source:https://get.ishadowx.net/#portfolio //https://get.ishadowx.net/#portfolio string url = "https://get.ishadowx.net/#portfolio"; var web1 = new HtmlWeb(); var doc1 = web1.LoadFromBrowser(url, o => { var webBrowser = (WebBrowser)o; // WAIT until the dynamic text is set return(!string.IsNullOrEmpty(webBrowser.Document.GetElementById("portfolio").InnerText)); }); string server = doc1.DocumentNode.SelectSingleNode("//span[@id='ip" + area + "']").InnerText.Trim(); string port = doc1.DocumentNode.SelectSingleNode("//span[@id='port" + area + "']").InnerText.Trim(); string password = doc1.DocumentNode.SelectSingleNode("//span[@id='pw" + area + "']").InnerText.Trim(); return(new ServerTinfo(server, port, password)); }
public static List <NameValue> Get91wanAreas() { if (areas_of_91wan != null) { if (DateTime.Now - last_91wan < TimeSpan.FromMinutes(10)) { return(areas_of_91wan); } } var web = new HtmlWeb(); //var doc = web.Load("http://frxz2.91wan.com/list/"); var doc = web.LoadFromBrowser("http://frxz2.91wan.com/list/"); last_91wan = DateTime.Now; areas_of_91wan = doc.DocumentNode.Descendants("ul").Where(y => y.GetAttributeValue("id", "") == "server_all").First().Descendants("a").Select(item => new NameValue() { Name = new Regex("(?<=】).*| ").Replace(item.LastChild.InnerText, ""), Value = item.GetAttributeValue("href", "") }).ToList(); return(areas_of_91wan); }
private static void Test(string url, int timeoutMins) { Console.Write($"{url} ... "); var web = new HtmlWeb { CaptureRedirect = true, UseCookies = false, UsingCache = false, BrowserTimeout = TimeSpan.FromMinutes(timeoutMins), UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" }; HtmlDocument doc; try { doc = web.LoadFromBrowser(url); Console.WriteLine($"{web.StatusCode} {doc.DocumentNode.OuterHtml.Length} bytes."); } catch (HtmlWebException HWE) { Console.Write($"LoadFromBrowser {HWE.Message}; "); if (HWE.InnerException != null) { Console.Write($"LoadFromBrowser {HWE.InnerException.Message}."); } Console.WriteLine(); File.AppendAllText("HAPBUG.TXT", $"{url}\t{HWE.Message}\r\n"); } catch (Exception E) { Console.Write($"LoadFromBrowser {E.Message}; "); if (E.InnerException != null) { Console.Write($"LoadFromBrowser {E.InnerException.Message}."); } Console.WriteLine(); File.AppendAllText("HAPBUG.TXT", $"{url}\t{E.Message}\r\n"); } }
/// <summary> /// Waits for specific items to loads and then returns the dynamically rendered webpage. /// </summary> /// <param name="htmlParser">Html parser.</param> /// <param name="webClient">Webclient that loads the dynamic content.</param> /// <param name="url">URL to load.</param> /// <param name="waitForTag">Wait for tag to load.</param> /// <returns>Loaded document.</returns> private static IHtmlDocument LoadDocument(HtmlParser htmlParser, HtmlWeb webClient, string url, string waitForTag) { bool result = false; IHtmlDocument document = null; var doc1 = webClient.LoadFromBrowser(url, o => { if (!result) { document = htmlParser.Parse(((WebBrowser)o).Document.Body.InnerHtml); var dynamicContent = document.QuerySelector(waitForTag); if (dynamicContent != null && dynamicContent.HasChildNodes) { result = true; } } return(result); }); result = false; return(document); }
/// <summary> /// Grabs the required trs from the market table after calculating the range from the base number. /// </summary> /// <param name="marketURL">The market URL</param> /// <param name="openMarketBaseNumber">The open market base number</param> /// <returns>HtmlNodeCollection</returns> private HtmlNodeCollection DownloadMarketData(string marketURL, int openMarketBaseNumber) { // Define the range baseNumber = Math.Round(Convert.ToDecimal(openMarketBaseNumber), 2); baseNumberPlus50 = baseNumber + 50; baseNumberPlus100 = baseNumber + 100; baseNumberPlus150 = baseNumber + 150; baseNumberPlus200 = baseNumber + 200; baseNumberMinus50 = baseNumber - 50; baseNumberMinus100 = baseNumber - 100; // Grab all rows var htmlWeb = new HtmlWeb(); HtmlAgilityPack.HtmlDocument htmlDocument = htmlWeb.LoadFromBrowser(marketURL); HtmlNodeCollection tableRows = htmlDocument.DocumentNode.SelectNodes("//table[@id=\"octable\"]//tr"); tableRows.RemoveAt(tableRows.Count - 1); tableRows.RemoveAt(0); tableRows.RemoveAt(0); // Get only those rows which contain values for the defined tange HtmlNodeCollection workSetRows = new HtmlNodeCollection(null); foreach (var currentTableRow in tableRows) { if (currentTableRow.InnerHtml.Contains(baseNumber.ToString()) || currentTableRow.InnerHtml.Contains(baseNumberPlus50.ToString()) || currentTableRow.InnerHtml.Contains(baseNumberPlus100.ToString()) || currentTableRow.InnerHtml.Contains(baseNumberMinus50.ToString()) || currentTableRow.InnerHtml.Contains(baseNumberMinus100.ToString()) || currentTableRow.InnerHtml.Contains(baseNumberPlus150.ToString()) || currentTableRow.InnerHtml.Contains(baseNumberPlus200.ToString())) { workSetRows.Add(currentTableRow); } } return(workSetRows); }
private async void Load() { if (string.IsNullOrWhiteSpace(BaseUrl.Value)) { return; } DocumentAvailable.Value = false; try { Nodes.Value = null; CurrentNode.Value = null; //HtmlRoot = await Web.LoadFromWebAsync(BaseUrl.Value); HtmlRoot = Web.LoadFromBrowser(BaseUrl.Value); if (null != HtmlRoot) { DocumentAvailable.Value = true; CurrentNode.Value = new DxxHtmlNode(HtmlRoot.DocumentNode); Nodes.Value = CurrentNode.Value.ChildNodes; } } catch (Exception e) { Debug.WriteLine(e.ToString()); DxxLogger.Instance.Error(LOG_CAT, e.Message); } }
private void Detalhes(object sender, EventArgs e) { DataGridViewRow row = new DataGridViewRow(); row = dataGridView1.SelectedRows[0]; string pagina = row.Cells[4].Value.ToString(); var web1 = new HtmlWeb(); var doc1 = web1.LoadFromBrowser(pagina, o => { var webBrowser = (WebBrowser)o; return(!string.IsNullOrEmpty(webBrowser.Document.GetElementById("threadPageContentContainer").InnerText)); }); var titulo = doc1.DocumentNode.SelectSingleNode("//span[@name='subject']").InnerText; var nome = doc1.DocumentNode.SelectSingleNode("//a[@class='profile-mini-display-name-link']").InnerText; var data = doc1.DocumentNode.SelectSingleNode("//div[@class='date']").InnerText; var pergunta = doc1.DocumentNode.SelectSingleNode("//div[@class='messageContent']").InnerText; //var pergunta = doc1.DocumentNode.SelectSingleNode("//ul[@id='rootMessage']").InnerText // .Replace("Entrar para Votar", "").Replace("Pergunta",""); //var respostas = doc1.DocumentNode.SelectSingleNode("//ul[@class='messageContent']").InnerText.Replace("Entrar para Votar",""); //var z = pergunta.SelectSingleNode("//div[@class='body']").InnerText; //var resposta = doc1.DocumentNode.Descendants().First(x => x.Attributes["class"] != null && // x.Attributes["class"].Value.Equals("message answer")).InnerText; Detalhar detalhar = new Detalhar(nome, titulo, data, pergunta); detalhar.Show(); Console.WriteLine(); }