private void parseBTN_Click(object sender, RoutedEventArgs e)
{
for (int j = 1; j <= Int32.Parse(countTB.Text); j++)
{
string url = linkTB.Text + "?page=" + j;
var web = new HtmlWeb();
var doc = web.LoadFromBrowser(url);
var names = doc.DocumentNode.SelectNodes("//li[@class = 'catalog__item inner ng-scope']//p[@class = 'catalog__name']");
var prices = doc.DocumentNode.SelectNodes("//li[@class = 'catalog__item inner ng-scope']//p[@class = 'catalog__price ng-scope ng-binding']");
for (int i = 0; i <= names.Count - 1; i++)
{
string priceNumbers = RemoveNonNumeric(prices[i].InnerText);
Plywood good = new Plywood
{
GoodName = names[i].InnerText.Trim(),
GoodPrice = Double.Parse(priceNumbers.Remove(priceNumbers.Length - 2)),
Height = ExtractDimensions(names[i].InnerText.Trim()).Split('-')[0],
Width = ExtractDimensions(names[i].InnerText.Trim()).Split('-')[1],
Depth = ExtractDimensions(names[i].InnerText.Trim()).Split('-')[2]
};
good.Square = Double.Parse(good.Height) * Double.Parse(good.Width) / 1000000;
good.GoodPriceM2 = good.GoodPrice / good.Square;
goodsDGRID.Items.Add(good);
}
}
}
static void Main(string[] args)
{
string url = "http://html-agility-pack/from-browser";
var web1 = new HtmlWeb();
var doc1 = web1.LoadFromBrowser(url, o =>
{
var webBrowser = (WebBrowser)o;
// WAIT until the dynamic text is set
return(!string.IsNullOrEmpty(webBrowser.Document.GetElementById("uiDynamicText").InnerText));
});
var t1 = doc1.DocumentNode.SelectSingleNode("//div[@id='uiDynamicText']").InnerText;
var web2 = new HtmlWeb();
var doc2 = web2.LoadFromBrowser(url, html =>
{
// WAIT until the dynamic text is set
return(!html.Contains("<div id=\"uiDynamicText\"></div>"));
});
var t2 = doc2.DocumentNode.SelectSingleNode("//div[@id='uiDynamicText']").InnerText;
Console.WriteLine("Text 1: " + t1);
Console.WriteLine("Text 2: " + t2);
}
public List <WebData> GetShopHiveDataAsync(string productName)
{
List <WebData> shophive = new List <WebData>();
try
{
var web2 = new HtmlWeb();
web2.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36";
web2.BrowserTimeout = TimeSpan.Zero;
var doc2 = web2.LoadFromBrowser(string.Format("http://www.shophive.com/catalogsearch/result/?cat=0&q={0}", productName), html =>
{
// WAIT until the dynamic text is set
return(!html.Contains("<div class=\"products-grid\"></div>"));
});
var t1 = doc2.DocumentNode.SelectNodes(".//div[@class='product-block']");
foreach (HtmlNode ht in t1)
{
shophive.Add(new WebData
{
Name = ht.SelectNodes(".//div[@class='product-block-inner']/a")[0].Attributes["title"].Value,
Link = ht.SelectNodes(".//div[@class='product-block-inner']/a")[0].Attributes["href"].Value,
Image = ht.SelectNodes(".//div[@class='product-block-inner']/a/img")[0].Attributes["src"].Value,
Price = ht.SelectNodes(".//div[@class='price-box']/span")[0].InnerText
});
}
}
catch (Exception ex)
{
shophive = null;
}
return(shophive);
}
public static HtmlDocument LoadPagefromBrower(string url)
{
HtmlWeb webAgent = new HtmlWeb();
HtmlDocument htmlDocument = webAgent.LoadFromBrowser(url);
return(htmlDocument);
}
public List <WebData> GetSymbiosDataAsync(string productName)
{
List <WebData> symbios = new List <WebData>();
try
{
var web2 = new HtmlWeb();
web2.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36";
web2.BrowserTimeout = TimeSpan.Zero;
var doc2 = web2.LoadFromBrowser(string.Format("https://www.symbios.pk/search.php?search_query={0}", productName), html =>
{
// WAIT until the dynamic text is set
return(!html.Contains("<div id=\"category-products\"></div>"));
});
int i = 0;
foreach (HtmlNode li in doc2.DocumentNode.SelectNodes("//div[contains(@class,'products')]"))
{
symbios.Add(new WebData()
{
Link = li.SelectNodes(@"//div[@class='product-info text-center']/h3[@class='name']/a")[i].Attributes["href"].Value,
Price = li.SelectNodes(@"//div[@class='product-info text-center']/div[@class='product-price']/span[@class='price']")[i].InnerText,
Image = li.SelectNodes(@"//div[@class='product-image']/a/img")[i].Attributes["src"].Value,
Name = li.SelectNodes(@"//div[@class='product-info text-center']/h3[@class='name']/a")[i].InnerText,
});
i++;
}
}
catch (Exception ex)
{
symbios = null;
}
return(symbios);
}
static void Main(string[] args)
{
const string SECTION_CLASS = "WyeMbd";
const string SECTION_HEADER_CLASS = "adH5zf";
const string MAIN_COVERAGE_CLASS = "nuEeue";
var csv = new StringBuilder();
string filePath = "headlines.csv";
var URL = @"https://news.google.com/";
HtmlWeb web = new HtmlWeb();
var htmlDoc = web.LoadFromBrowser(URL);
var frontpageNewsSections = htmlDoc.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains(SECTION_CLASS));
List <string> headers = new List <string>();
List <List <string> > allMainHeadlines = new List <List <string> >();
foreach (var section in frontpageNewsSections)
{
var header = section.Descendants().Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains(SECTION_HEADER_CLASS)).First().InnerText;
var mainHeadlines = section.Descendants().Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains(MAIN_COVERAGE_CLASS));
headers.Add(header);
Console.WriteLine(header);
allMainHeadlines.Add(new List <string>());
foreach (var headline in mainHeadlines)
{
var santizedHeadline = WebUtility.HtmlDecode(headline.InnerText);
allMainHeadlines.Last().Add(santizedHeadline);
Console.WriteLine("\t" + santizedHeadline);
}
}
foreach (var header in headers)
{
csv.Append(header + ",");
}
csv.Append(Environment.NewLine);
int maxHeadLines = allMainHeadlines.Select(x => x.Count).ToList().Max();
for (int i = 0; i < maxHeadLines; i++)
{
foreach (var sectionHeadlines in allMainHeadlines)
{
if (i < sectionHeadlines.Count)
{
csv.Append('"' + sectionHeadlines[i] + '"' + ",");
}
}
csv.Append(Environment.NewLine);
}
File.WriteAllText(filePath, csv.ToString());
Console.WriteLine();
Console.WriteLine("Output exported to " + filePath);
Console.ReadKey();
}
public List <WebData> GetYayvoDataAsync(string productName)
{
List <WebData> yayvo = new List <WebData>();
try
{
var web2 = new HtmlWeb();
web2.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36";
web2.BrowserTimeout = TimeSpan.Zero;
var doc2 = web2.LoadFromBrowser(string.Format("http://yayvo.com/search/result/?q={0}", productName), html =>
{
// WAIT until the dynamic text is set
return(!html.Contains("<ul class=\"products-grid\"></ul>"));
});
var liList = doc2.DocumentNode.SelectNodes("//ul[contains(@class, 'products-grid')]");
foreach (HtmlNode li in liList)
{
yayvo.Add(new WebData()
{
Link = li.Descendants("a").First().Attributes["href"].Value,
Name = li.Descendants("a").First().Attributes["title"].Value,
Image = li.Descendants("a").First().Descendants("img").First().Attributes["src"].Value,
Price = li.SelectSingleNode(@"//div[@class='price-box']/p[@class='special-price']/span").InnerText
});
}
}
catch (Exception ex)
{
yayvo = null;
}
return(yayvo);
}
private void button1_Click(object sender, EventArgs e)
{
HtmlWeb hw = new HtmlWeb();
HtmlAgilityPack.HtmlDocument doc = hw.LoadFromBrowser("Insert URL");
// foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a")){ Debug.WriteLine(link.Attributes["href"].Value); }
// string[] Links = new string[3015]; // The Total
var ssa = 0;
foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]"))
{
//Debug.WriteLine(link.InnerText);
if (link.InnerText.Contains(".jpg") || link.InnerText.Contains(".png") || link.InnerText.Contains(".gif") || link.InnerText.Contains(".jpeg"))
{
treeView1.Nodes.Add(link.Attributes["href"].Value);
// Links[ssa] = link.InnerText;
}
ssa++;
Debug.WriteLine(link.Id.ToString());
//if (link.Name) { }
// Debug.WriteLine(link.);
}
Debug.WriteLine(ssa);
Debug.WriteLine(treeView1.Nodes.Count);
}
private List <String> downloadAndProcessKaduEpisodes(String url)
{
HtmlWeb web = new HtmlWeb();
var htmlDoc = web.LoadFromBrowser(url);
var nodes = htmlDoc.DocumentNode.SelectNodes("//div[@id='video-line-content']/a");
return(nodes.Select(x => "http://kadu.ru" + x.Attributes["href"].Value).ToList());
}
private HtmlDocument GetHtmlDocument(string url, TimeSpan outTime, Func <string, bool> isScriptCompleted = null)
{
try
{
var htmlWeb = new HtmlWeb()
{
BrowserTimeout = outTime,
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36",
CaptureRedirect = true
};
return(isScriptCompleted == null?
htmlWeb.LoadFromBrowser(url) : htmlWeb.LoadFromBrowser(url, isScriptCompleted));
}
catch (Exception ex)
{
throw ex;
}
}
private HtmlBelge Indir()
{
HtmlWeb web = new HtmlWeb
{
BrowserTimeout = new TimeSpan(0)
};
HtmlBelge htmlBelgeYukle = web.LoadFromBrowser(SecilenSunucuAdresi());
return(htmlBelgeYukle);
}
public string GetLyric(string songName, string artist = null)
{
if (InternetConnection.IsConnectedToInternet() == false)
{
return(string.Empty);
}
if (string.IsNullOrWhiteSpace(artist))
{
document = html.LoadFromBrowser("https://mp3.zing.vn/tim-kiem/bai-hat.html?q=" + songName);
}
else
{
document = html.LoadFromBrowser("https://mp3.zing.vn/tim-kiem/bai-hat.html?q=" + songName + ' ' + artist);
}
string link = null;
try
{
var data = document.DocumentNode.QuerySelector(".fl .title-song h3 a");
link = @"https://mp3.zing.vn" + data.Attributes["href"].DeEntitizeValue;
document = html.LoadFromBrowser(link);
var lyric = document.DocumentNode.QuerySelector(".fn-container [id] p");
document = null;
if (lyric == null)
{
return(string.Empty);
}
else
{
return(lyric.InnerHtml.Replace("<br>", "\n"));
}
}
catch
{
document = null;
return(string.Empty);
}
finally
{
GC.Collect();
}
}
public void GetAllHrefs(String uri)
{
var web = new HtmlWeb();
web.BrowserTimeout = TimeSpan.FromSeconds(30);
var doc = web.LoadFromBrowser(uri, o =>
{
// var webBrowser = (WebBrowser)o;
return(o.Contains("card-content-title"));
});
GC.Collect();
// Show results
Regex re = new Regex(@">(.*?)<");
foreach (var title in doc.DocumentNode.SelectNodes(".//div[@class='card-content']"))
{
if (title.SelectSingleNode("a[@class='card-content-title']") != null)
{
string url = "https://docs.microsoft.com/en-us" + title.SelectSingleNode("a[@class='card-content-title']").GetAttributeValue("href", String.Empty);
hrefslist.Add(url);
if (title.SelectSingleNode("ul[@class='tags']") != null)
{
string result = "";
string a = (title.SelectSingleNode("ul[@class='tags']").InnerText);
foreach (Match match in re.Matches(a))
{
if (match.Groups[1].Length != 0)
{
result = match.Groups[1].Value;
break;
}
/* if(!match.Groups[1].Equals("Advanced") && !match.Groups[1].Equals("Intermediate") && !match.Groups[1].Equals("Beginner") )
* {
* result += match.Groups[1] + ", ";
* } */
}
// if(result.Length>2) result = result.Substring(0, result.Length - 2);
prodDictionary[url] = result;
}
}
}
}
public static void PopulateCategory(Mouser.Domain.ApplicationContext context)
{
try
{
string currentUri = "https://www2.mouser.com/Electronic-Components/";
var web = new HtmlWeb();
var doc = web.LoadFromBrowser(HttpUtility.HtmlDecode(currentUri), o =>
{
WebBrowser webBrowser = (WebBrowser)o;
// WAIT until the dynamic text is set
return(!string.IsNullOrEmpty(webBrowser.Document.GetElementById("mlnkMailTo").InnerText));
});
var webCategories = new List <WebCategory>();
var categoryNodes = doc.DocumentNode.SelectNodes("//*[@class=\"panel light-grey-panel\"]");
foreach (var node in categoryNodes)
{
var categoryName = node.SelectSingleNode(".//h2[1]").InnerText;
var subCategoryNameNodes = node.SelectNodes(".//a[@class=\"SearchResultsSubLevelCategory\"]");
foreach (var subCategoryName in subCategoryNameNodes)
{
webCategories.Add(new WebCategory
{
Category = categoryName.Replace("&", "&"),
SubCategory = subCategoryName.InnerText.Substring(0, subCategoryName.InnerText.LastIndexOf(" ") - 1).Replace("&", "&")
});
}
}
foreach (var wc in webCategories)
{
List <Category> categories = context.Categories.Include(c => c.Manufacturer).Where(c => !c.IsCategory && c.Name == wc.SubCategory).ToList();
foreach (Category category in categories)
{
var cat = context.Categories.FirstOrDefault(c => c.IsCategory && c.Name == wc.Category && c.Manufacturer.Id == category.Manufacturer.Id);
if (cat == null)
{
cat = new Category {
IsCategory = true, Manufacturer = category.Manufacturer, Name = wc.Category
};
context.Categories.Add(cat);
context.SaveChanges();
}
category.ParentId = cat.Id;
context.SaveChanges();
}
}
}
finally
{
}
}
static void Main(string[] args)
{
// 348050423: 剑如长虹气如玉
_activeUserId = ConfigurationManager.AppSettings["UserId"];
if (_activeUserId == null)
{
throw new Exception("AppSettings未配置UserId");
}
var firstPageUrl = GetVideoUrl(_activeUserId, 1);
_web = new HtmlWeb();
var document = _web.LoadFromBrowser(firstPageUrl, o =>
{
var webBrowser = (WebBrowser)o;
// WAIT until the dynamic text is set
return(webBrowser.Document.GetElementById("submit-video-list") != null && webBrowser.Document.GetElementById("submit-video-list").Children[1].Children.Count > 0);
});
var pageNumNode = document.DocumentNode.SelectSingleNode(".//div[@id='submit-video-list']/ul[@class='be-pager']/span[@class='be-pager-total']");
// 总页数
var pageCount = GetPageCount(pageNumNode);
var totalCountNode = document.DocumentNode.SelectSingleNode(".//div[@id='submit-video-type-filter']/a[1]/span");
var totalCount = totalCountNode.InnerText;
Console.WriteLine($"总页数: { pageCount}, 总个数: {totalCount}");
if (pageCount <= 0)
{
return;
}
StringBuilder result = new StringBuilder(500);
result.AppendLine($"总数: {totalCount}, 生成时间{DateTime.Now}");
result.AppendLine(HEADER);
for (int i = 1; i <= pageCount; i++)
{
Console.WriteLine($"第{i}页读取中...");
var videoInfo = GetVideosFromNode(i);
result.Append(videoInfo);
}
System.IO.File.WriteAllText($"export-{_activeUserId}.csv", result.ToString());
Console.WriteLine("Load Complete");
}
private void simpleButton6_Click(object sender, EventArgs e)
{
var web2 = new HtmlWeb();
string currentUri = @"https://ru.mouser.com/ProductDetail/TE-Connectivity/M85049-93-16?qs=2WXlatMagcH45qgLJ7jojA%3D%3D";
var doc2 = web2.LoadFromBrowser(HttpUtility.HtmlDecode(currentUri), o =>
{
WebBrowser webBrowser = (WebBrowser)o;
// WAIT until the dynamic text is set
return(!string.IsNullOrEmpty(webBrowser.Document.GetElementById("mlnkMailTo").InnerText));
});
int sds = 3;
return;
try
{
Console.WriteLine(String.Format("*** Begin Request ***"));
var request = (System.Net.HttpWebRequest)System.Net.WebRequest.Create("http://api.PhantomJScloud.com/api/browser/v2/a-demo-key-with-low-quota-per-ip-address/");
request.ContentType = "application/json";
request.Method = "POST";
request.Timeout = 45000; //45 seconds
request.KeepAlive = false;
request.MediaType = "application/json";
request.ServicePoint.Expect100Continue = false; //REQUIRED! or you will get 502 Bad Gateway errors
using (var streamWriter = new System.IO.StreamWriter(request.GetRequestStream()))
{
//you should look at the HTTP Endpoint docs, section about "userRequest" and "pageRequest"
//for a listing of all the parameters you can pass via the "pageRequestJson" variable.
string pageRequestJson = @"{'url':'https://ru.mouser.com/ProductDetail/TE-Connectivity/M85049-93-16?qs=2WXlatMagcH45qgLJ7jojA%3D%3D','renderType':'plainText','outputAsJson':true }";
streamWriter.Write(pageRequestJson);
streamWriter.Flush();
streamWriter.Close();
}
var response = (System.Net.HttpWebResponse)request.GetResponse();
Console.WriteLine(String.Format("HttpWebResponse.StatusDescription: {0}", response.StatusDescription));
using (var streamReader = new System.IO.StreamReader(response.GetResponseStream()))
{
string server_reply = streamReader.ReadToEnd();
Console.WriteLine(String.Format("Server Response content: {0}", server_reply));
}
response.Close();
}
catch (Exception Ex)
{
Console.WriteLine("*** HTTP Request Error ***");
Console.WriteLine(Ex.Message);
}
}
private String downloadKaduVideoToFolder(String url, String saveToPath)
{
HtmlWeb web = new HtmlWeb();
var htmlDoc = web.LoadFromBrowser(url);
var node = htmlDoc.DocumentNode.SelectSingleNode("//video[@id='video-flash']");
var videoUrl = node.Attributes["src"].Value;
var titleNode = htmlDoc.DocumentNode.SelectSingleNode("//h1");
var title = titleNode.InnerHtml;
int t = title.IndexOf("</span>");
title = title.Substring(t, title.Length - t) + ".mp4";
using (var webClient = new WebClient())
{
webClient.DownloadFile(videoUrl, title);
}
return(title);
}
public string GetHtmlDocument(string url)
{
var webPage = new HtmlWeb();
HtmlDocument pageContents = new HtmlDocument();
try
{
pageContents = webPage.LoadFromBrowser(url);
}
catch (Exception e)
{
Console.WriteLine("Sorry that was an invalid url. Please enter a valid one.");
string newUrl = Console.ReadLine();
GetHtmlDocument(newUrl);
}
return(pageContents.ParsedText);
}
public static List <string> GetGoogleLinks(string URl, out bool done)
{
List <string> rtn = new List <string>();
try
{
var web = new HtmlWeb();
var xPath = @"//div[contains(concat(' ', normalize-space(@class), ' '), ' r ')]";
var doc = web.LoadFromBrowser(URl);
var nodes = doc.DocumentNode.SelectNodes(xPath);
foreach (var item in nodes)
{
var url = item.ChildNodes.FindFirst("a").Attributes["href"].Value;
if (url.StartsWith("http"))
{
try
{
doc = web.Load(url);
var Nodes = GetNodes(doc);
if (Nodes != null)
{
rtn.Add(url);
}
}
catch (Exception)
{
}
}
}
}
catch (Exception ex)
{
Xceed.Wpf.Toolkit.MessageBox.Show("An error occured. \n Please check your internet connection and try again.\n" + ex.Message);
}
finally
{
done = true;
}
return(rtn);
}
/// <summary>
/// Returns the Bank NIFTY page URL from the main market URL JavaScript snippet.
/// </summary>
/// <param name="marketURL">The market URL for Bank NIFTY.</param>
/// <returns>The Bank NIFTY URL.</returns>
private string GetBankNIFTYMarketURL(string marketURL)
{
// Load the web page and get the JavaScript
var htmlWeb = new HtmlWeb();
HtmlAgilityPack.HtmlDocument htmlDocument = htmlWeb.LoadFromBrowser(marketURL);
HtmlNodeCollection scriptTags = htmlDocument.DocumentNode.SelectNodes("//script[@type=\"text/javascript\"]");
string bankNIFTYMarketURL = scriptTags[17].InnerHtml;
// Process the JavaScript and get the Bank NIFTY URL
bankNIFTYMarketURL = bankNIFTYMarketURL.Replace("\r\n", "");
bankNIFTYMarketURL = bankNIFTYMarketURL.Replace("\t", "");
bankNIFTYMarketURL = bankNIFTYMarketURL.Remove(0, bankNIFTYMarketURL.IndexOf("BANKNIFTY"));
bankNIFTYMarketURL = bankNIFTYMarketURL.Remove(0, 28);
bankNIFTYMarketURL = bankNIFTYMarketURL.Remove(bankNIFTYMarketURL.IndexOf(";"), bankNIFTYMarketURL.Length - bankNIFTYMarketURL.IndexOf(";"));
bankNIFTYMarketURL = bankNIFTYMarketURL.Replace("'", "");
return(NSEIndiaWebsiteURL + "/" + bankNIFTYMarketURL);
}
public static void CheckForUpdate()
{
if (File.Exists("AtlusScriptCompiler.exe"))
{
string version = "";
string download = "";
// Load webpage in browser
HtmlWeb web = new HtmlWeb();
web.BrowserTimeout = TimeSpan.FromSeconds(0);
var appveyorPage = web.LoadFromBrowser("https://ci.appveyor.com/project/TGEnigma/atlusscripttools/build/artifacts", o =>
{
var webBrowser = (WebBrowser)o;
// Wait until the download url loads
return(webBrowser.Document.Body.InnerHtml.Contains("artifact-type"));
});
// Scrape version and download url from HTML
version = appveyorPage.DocumentNode.SelectSingleNode("//div[@class='project-build-version ng-binding']").InnerText;
download = appveyorPage.DocumentNode.SelectSingleNode("//a[@class='artifact-type zip']").Attributes["href"].Value;
if (version != "" && download != "")
{
// Create/Modify Update XML
File.WriteAllText("updates.xml", $"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<item>\n\t<version>{version}.0</version>\n\t<url>{download}</url>\n\t<changelog>https://github.com/TGEnigma/Atlus-Script-Tools/commits/master</changelog>\n\t<mandatory>false</mandatory>\n</item>");
//Wait for XML file to exist/not be in use
using (WaitForFile("updates.xml", FileMode.Open, FileAccess.ReadWrite, FileShare.None)) { };
// Ask to download and install update from XML data
AutoUpdater.RunUpdateAsAdmin = false;
AutoUpdater.InstalledVersion = new Version(AssemblyName.GetAssemblyName("AtlusScriptCompiler.exe").Version.ToString());
AutoUpdater.Start("updates.xml");
}
}
else
{
MessageBox.Show("Could not find AtlusScriptCompiler.exe. " +
"Put this program in the same folder and try running it again!",
"Critical Error", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
}
}
public TabControl RenderizeSeasonTab(string lastSeasonUrl, SearchSources sourceName)
{
HtmlAgilityPack.HtmlDocument htmldoc = _web.LoadFromBrowser(lastSeasonUrl);
List <ISourceScraperData> scraperData = new List <ISourceScraperData>();
IEnumerable <HtmlNode> episodes;
switch (sourceName)
{
case SearchSources.TuSubtitulo:
episodes = htmldoc.DocumentNode.Descendants("table").Skip(1);
foreach (HtmlNode episode in episodes)
{
TuSubtituloScraperData scraperItem = new TuSubtituloScraperData();
scraperItem.SetEpisodeData(episode, "https:");
scraperData.Add(scraperItem);
}
break;
case SearchSources.Subtitulamos:
HtmlNode episodesWrapper = htmldoc.DocumentNode.Descendants("div").SingleOrDefault(x => x.Id == "episode-choices");
episodes = episodesWrapper.Descendants("a");
foreach (HtmlNode episode in episodes)
{
HtmlAgilityPack.HtmlDocument episodeHtml = _web.Load("https://www.subtitulamos.tv" + episode.Attributes["href"].Value);
SubtitulamosScraperData scraperItem = new SubtitulamosScraperData();
scraperItem.SetEpisodeData(episodeHtml.DocumentNode.Descendants("div").SingleOrDefault(e => e.Attributes.Contains("class") && e.Attributes["class"].Value.Equals("content limited-width")), "https://www.subtitulamos.tv");
scraperData.Add(scraperItem);
}
break;
default:
MessageBox.Show("No se pudo generar los resultados. El servidor de datos para la búsqueda es incorrecta.");
break;
}
_sourceName = sourceName;
_TabCtrlResults = SetControls(scraperData, _TabCtrlResults);
return(_TabCtrlResults);
}
static string GetVideosFromNode(int pageNum)
{
var videoUrl = GetVideoUrl(_activeUserId, pageNum);
//var document = _web.LoadFromBrowser(videoUrl);
var document = _web.LoadFromBrowser(videoUrl, o =>
{
var webBrowser = (WebBrowser)o;
// WAIT until the dynamic text is set
return(webBrowser.Document.GetElementById("submit-video-list") != null && webBrowser.Document.GetElementById("submit-video-list").Children[1].Children.Count > 0);
});
// 第一个ul是详细模式, 第二个ul是表格模式
var videoNodes = document.DocumentNode.SelectNodes(".//div[@id='submit-video-list']/ul[2]/li");
if (videoNodes == null)
{
System.Console.WriteLine("no nodes");
return(string.Empty);
}
StringBuilder result = new StringBuilder();
foreach (var item in videoNodes)
{
var titleNode = item.SelectSingleNode(".//a[@class='title']");
var title = titleNode.Attributes["title"].Value;
var url = "https:" + titleNode.Attributes["href"].Value;
var videoTimeNode = item.SelectSingleNode(".//a[@class='cover']/span[@class='length']");
var videoTime = videoTimeNode.InnerText;
var metaNode = item.SelectSingleNode(".//div[@class='meta']/span[@class='time']");
var createTime = metaNode.InnerText.TrimEnd(' ', '\n');
result.AppendLine($"{title}\t{videoTime}\t{url}\t{createTime}\t{pageNum}");
}
return(result.ToString());
}
public static ServerTinfo GetSSbyArea(string area)
{
//view-source:https://get.ishadowx.net/#portfolio
//https://get.ishadowx.net/#portfolio
string url = "https://get.ishadowx.net/#portfolio";
var web1 = new HtmlWeb();
var doc1 = web1.LoadFromBrowser(url, o =>
{
var webBrowser = (WebBrowser)o;
// WAIT until the dynamic text is set
return(!string.IsNullOrEmpty(webBrowser.Document.GetElementById("portfolio").InnerText));
});
string server = doc1.DocumentNode.SelectSingleNode("//span[@id='ip" + area + "']").InnerText.Trim();
string port = doc1.DocumentNode.SelectSingleNode("//span[@id='port" + area + "']").InnerText.Trim();
string password = doc1.DocumentNode.SelectSingleNode("//span[@id='pw" + area + "']").InnerText.Trim();
return(new ServerTinfo(server, port, password));
}
public static List <NameValue> Get91wanAreas()
{
if (areas_of_91wan != null)
{
if (DateTime.Now - last_91wan < TimeSpan.FromMinutes(10))
{
return(areas_of_91wan);
}
}
var web = new HtmlWeb();
//var doc = web.Load("http://frxz2.91wan.com/list/");
var doc = web.LoadFromBrowser("http://frxz2.91wan.com/list/");
last_91wan = DateTime.Now;
areas_of_91wan = doc.DocumentNode.Descendants("ul").Where(y => y.GetAttributeValue("id", "") == "server_all").First().Descendants("a").Select(item => new NameValue()
{
Name = new Regex("(?<=】).*| ").Replace(item.LastChild.InnerText, ""),
Value = item.GetAttributeValue("href", "")
}).ToList();
return(areas_of_91wan);
}
private static void Test(string url, int timeoutMins)
{
Console.Write($"{url} ... ");
var web = new HtmlWeb
{
CaptureRedirect = true,
UseCookies = false,
UsingCache = false,
BrowserTimeout = TimeSpan.FromMinutes(timeoutMins),
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
};
HtmlDocument doc;
try
{
doc = web.LoadFromBrowser(url);
Console.WriteLine($"{web.StatusCode} {doc.DocumentNode.OuterHtml.Length} bytes.");
}
catch (HtmlWebException HWE)
{
Console.Write($"LoadFromBrowser {HWE.Message}; ");
if (HWE.InnerException != null)
{
Console.Write($"LoadFromBrowser {HWE.InnerException.Message}.");
}
Console.WriteLine();
File.AppendAllText("HAPBUG.TXT", $"{url}\t{HWE.Message}\r\n");
}
catch (Exception E)
{
Console.Write($"LoadFromBrowser {E.Message}; ");
if (E.InnerException != null)
{
Console.Write($"LoadFromBrowser {E.InnerException.Message}.");
}
Console.WriteLine();
File.AppendAllText("HAPBUG.TXT", $"{url}\t{E.Message}\r\n");
}
}
/// <summary>
/// Waits for specific items to loads and then returns the dynamically rendered webpage.
/// </summary>
/// <param name="htmlParser">Html parser.</param>
/// <param name="webClient">Webclient that loads the dynamic content.</param>
/// <param name="url">URL to load.</param>
/// <param name="waitForTag">Wait for tag to load.</param>
/// <returns>Loaded document.</returns>
private static IHtmlDocument LoadDocument(HtmlParser htmlParser, HtmlWeb webClient, string url, string waitForTag)
{
bool result = false;
IHtmlDocument document = null;
var doc1 = webClient.LoadFromBrowser(url, o =>
{
if (!result)
{
document = htmlParser.Parse(((WebBrowser)o).Document.Body.InnerHtml);
var dynamicContent = document.QuerySelector(waitForTag);
if (dynamicContent != null && dynamicContent.HasChildNodes)
{
result = true;
}
}
return(result);
});
result = false;
return(document);
}
/// <summary>
/// Grabs the required trs from the market table after calculating the range from the base number.
/// </summary>
/// <param name="marketURL">The market URL</param>
/// <param name="openMarketBaseNumber">The open market base number</param>
/// <returns>HtmlNodeCollection</returns>
private HtmlNodeCollection DownloadMarketData(string marketURL, int openMarketBaseNumber)
{
// Define the range
baseNumber = Math.Round(Convert.ToDecimal(openMarketBaseNumber), 2);
baseNumberPlus50 = baseNumber + 50;
baseNumberPlus100 = baseNumber + 100;
baseNumberPlus150 = baseNumber + 150;
baseNumberPlus200 = baseNumber + 200;
baseNumberMinus50 = baseNumber - 50;
baseNumberMinus100 = baseNumber - 100;
// Grab all rows
var htmlWeb = new HtmlWeb();
HtmlAgilityPack.HtmlDocument htmlDocument = htmlWeb.LoadFromBrowser(marketURL);
HtmlNodeCollection tableRows = htmlDocument.DocumentNode.SelectNodes("//table[@id=\"octable\"]//tr");
tableRows.RemoveAt(tableRows.Count - 1);
tableRows.RemoveAt(0);
tableRows.RemoveAt(0);
// Get only those rows which contain values for the defined tange
HtmlNodeCollection workSetRows = new HtmlNodeCollection(null);
foreach (var currentTableRow in tableRows)
{
if (currentTableRow.InnerHtml.Contains(baseNumber.ToString()) || currentTableRow.InnerHtml.Contains(baseNumberPlus50.ToString()) ||
currentTableRow.InnerHtml.Contains(baseNumberPlus100.ToString()) || currentTableRow.InnerHtml.Contains(baseNumberMinus50.ToString()) ||
currentTableRow.InnerHtml.Contains(baseNumberMinus100.ToString()) || currentTableRow.InnerHtml.Contains(baseNumberPlus150.ToString()) ||
currentTableRow.InnerHtml.Contains(baseNumberPlus200.ToString()))
{
workSetRows.Add(currentTableRow);
}
}
return(workSetRows);
}
private async void Load()
{
if (string.IsNullOrWhiteSpace(BaseUrl.Value))
{
return;
}
DocumentAvailable.Value = false;
try {
Nodes.Value = null;
CurrentNode.Value = null;
//HtmlRoot = await Web.LoadFromWebAsync(BaseUrl.Value);
HtmlRoot = Web.LoadFromBrowser(BaseUrl.Value);
if (null != HtmlRoot)
{
DocumentAvailable.Value = true;
CurrentNode.Value = new DxxHtmlNode(HtmlRoot.DocumentNode);
Nodes.Value = CurrentNode.Value.ChildNodes;
}
} catch (Exception e) {
Debug.WriteLine(e.ToString());
DxxLogger.Instance.Error(LOG_CAT, e.Message);
}
}
private void Detalhes(object sender, EventArgs e)
{
DataGridViewRow row = new DataGridViewRow();
row = dataGridView1.SelectedRows[0];
string pagina = row.Cells[4].Value.ToString();
var web1 = new HtmlWeb();
var doc1 = web1.LoadFromBrowser(pagina, o =>
{
var webBrowser = (WebBrowser)o;
return(!string.IsNullOrEmpty(webBrowser.Document.GetElementById("threadPageContentContainer").InnerText));
});
var titulo = doc1.DocumentNode.SelectSingleNode("//span[@name='subject']").InnerText;
var nome = doc1.DocumentNode.SelectSingleNode("//a[@class='profile-mini-display-name-link']").InnerText;
var data = doc1.DocumentNode.SelectSingleNode("//div[@class='date']").InnerText;
var pergunta = doc1.DocumentNode.SelectSingleNode("//div[@class='messageContent']").InnerText;
//var pergunta = doc1.DocumentNode.SelectSingleNode("//ul[@id='rootMessage']").InnerText
// .Replace("Entrar para Votar", "").Replace("Pergunta","");
//var respostas = doc1.DocumentNode.SelectSingleNode("//ul[@class='messageContent']").InnerText.Replace("Entrar para Votar","");
//var z = pergunta.SelectSingleNode("//div[@class='body']").InnerText;
//var resposta = doc1.DocumentNode.Descendants().First(x => x.Attributes["class"] != null &&
// x.Attributes["class"].Value.Equals("message answer")).InnerText;
Detalhar detalhar = new Detalhar(nome, titulo, data, pergunta);
detalhar.Show();
Console.WriteLine();
}
