/// <summary> /// Creates an HTML node from a string representing literal HTML. /// </summary> /// <param name="html">The HTML text.</param> /// <returns>The newly created node instance.</returns> public static HtmlNode CreateNode(string html) { // REVIEW: this is *not* optimum... HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(html); return doc.DocumentNode.FirstChild; }
public override void MakeReady(int sectionId) { if (!this.User.IsLoggedIn) return; AsyncHelper.Run(() => { string url = this.BaseUrl + this.NewTopicPath.Replace("{0}", sectionId.ToString()); var doc = new HtmlDocument(); var replacements = new Dictionary<string, string>(); HttpWebRequest req = Http.Prepare(url); req.Method = "GET"; HttpResult result = this.AllowRedirects ? Http.HandleRedirects(Http.Request(req), false) : Http.Request(req); // Did the request fail? if (result.HasError || result.Data.Trim().Length == 0) { ErrorLog.LogException(result.Error); this.OnReadyChanged(this, new EventArgs()); return; } doc.LoadHtml(result.Data); // Extract required data foreach (HtmlNode n in doc.DocumentNode.SelectNodes("//input[@type='hidden']")) { switch (n.Attributes["name"].Value) { case "auth_key": replacements.Add("[auth_key]", n.Attributes["value"].Value); break; case "attach_post_key": replacements.Add("[attach_post_key]", n.Attributes["value"].Value); break; case "s": replacements.Add("[s]", n.Attributes["value"].Value); break; default: break; } } // Check if we got the needed info if (replacements.Count != 3) replacements.Clear(); // Done this.TemplateReplacements = replacements; this.OnReadyChanged(this, new EventArgs()); if (!this.IsReady) { var error = new Exception(String.Format( "MakeReady({0}) failed for site '{1}'.\r\nUrl used: {2}.", sectionId, this.BaseUrl, url )); ErrorLog.LogException(error); } }); }
public static HtmlDocument GetDoc(string url) { var doc = new HtmlDocument(); doc.LoadHtml(Get(url).Data); return doc; }
public override SiteTopic GetTopic(string url) { if (!this.User.IsLoggedIn) return null; HtmlDocument doc = new HtmlDocument(); HttpWebRequest req; HttpResult result; req = Http.Prepare(url); req.Method = "GET"; req.Referer = url; try { result = this.AllowRedirects ? Http.HandleRedirects(Http.Request(req), false) : Http.Request(req); doc.LoadHtml(result.Data); ErrorLog.LogException(result.Error); HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes("//a[@title='Reply directly to this post']"); string link = HttpUtility.HtmlDecode(nodes[0].GetAttributeValue("href", String.Empty)); req = Http.Prepare(link); req.Method = "GET"; req.Referer = url; result = this.AllowRedirects ? Http.HandleRedirects(Http.Request(req), false) : Http.Request(req); doc.LoadHtml(result.Data); ErrorLog.LogException(result.Error); string title = (from n in doc.DocumentNode.SelectNodes("//h2[@class='maintitle']") where n.InnerText.Trim().Contains("Replying to ") select n.InnerText.Replace("Replying to ", "")).ToArray()[0]; string content = doc.DocumentNode.SelectNodes("//textarea[@name='Post']")[0].InnerText; content = HttpUtility.HtmlDecode(content.Substring(content.IndexOf(']') + 1)).Trim(); content = content.Substring(0, content.Length - "[/quote]".Length); // Fix IPB3 quotes string pattern = @"(?i)\[quote [\w\d " + '"' + @"'-=]+\]"; string replace = "[quote]"; content = Regex.Replace(content, pattern, replace); return new SiteTopic( HttpUtility.HtmlDecode(title).Trim(), content.Trim(), 0, 0, url ); } catch (Exception error) { ErrorLog.LogException(error); return null; } }
public override string[] GetTopicUrls(string html) { if (html == null || html.Trim().Length == 0) return new string[0]; var urls = new List<string>(); var doc = new HtmlDocument(); HtmlNodeCollection nodes; Uri uri; try { doc.LoadHtml(html); nodes = doc.DocumentNode.SelectNodes("//a[@class='topic_title']"); var anchors = from n in nodes where !n.ParentNode.InnerHtml.Contains("class=\"topic_prefix\"") && n.GetAttributeValue("id", "").StartsWith("tid-link-") && n.GetAttributeValue("href", "").StartsWith("http:") select HttpUtility.HtmlDecode(n.GetAttributeValue("href", "")).Trim(); //throw new Exception(anchors.ToArray().Length.ToString()); foreach (string a in anchors) { if (Uri.TryCreate(a, UriKind.Absolute, out uri)) urls.Add(a); } return urls.ToArray(); } catch (Exception error) { ErrorLog.LogException(error); return new string[0]; } }
public override string[] GetTopicUrls(string html) { if (html == null || html.Trim().Length == 0) return new string[0]; var urls = new List<string>(); var doc = new HtmlDocument(); HtmlNodeCollection nodes; try { doc.LoadHtml(html); nodes = doc.DocumentNode.SelectNodes("//a"); var anchors = from n in nodes where n.GetAttributeValue("id", "").StartsWith("thread_title_") && !n.ParentNode.InnerHtml.Contains("Sticky:") select HttpUtility.HtmlDecode(n.GetAttributeValue("href", "")).Trim(); //throw new Exception(anchors.ToArray().Length.ToString()); foreach (string a in anchors) { if (a.Length > 0) urls.Add((a.StartsWith("http:")) ? a : this.BaseUrl + "/" + a); } return urls.ToArray(); } catch (Exception error) { ErrorLog.LogException(error); return new string[0]; } }
public override string[] GetTopicUrls(string html) { if (html == null || html.Trim().Length == 0) return new string[0]; var allowAdd = true; var urls = new List<string>(); var doc = new HtmlDocument(); HtmlNodeCollection nodes; Uri uri; try { doc.LoadHtml(html); nodes = doc.DocumentNode.SelectNodes("//table"); var table = (from n in nodes where n.InnerHtml.Contains("id=\"tid-link-") && n.InnerHtml.Contains("topic_toggle_folder") && n.InnerHtml.Contains("<!-- Begin Topic Entry ") select n).ToArray(); //throw new Exception(table[table.Length - 1].SelectNodes(".//td[1]").Count.ToString()); foreach (var n in table[table.Length - 1].SelectNodes(".//td[1]")) { if (n.InnerHtml.Contains("<b>Forum Topics</b>")) { allowAdd = false; break; } } foreach (var n in table[table.Length - 1].SelectNodes(".//tr")) { if (!allowAdd) { nodes = n.SelectNodes(".//td[1]"); if (nodes.Count > 0 && nodes[0].InnerHtml.Contains("<b>Forum Topics</b>")) { allowAdd = true; } } else { nodes = n.SelectNodes(".//a"); if (nodes.Count > 0) { var links = (from link in nodes where link.GetAttributeValue("id", "").StartsWith("tid-link-") && link.GetAttributeValue("href", "").StartsWith("http:") select HttpUtility.HtmlDecode(link.GetAttributeValue("href", ""))). ToArray(); if (links.Length > 0 && Uri.TryCreate(links[0].Trim(), UriKind.Absolute, out uri)) { urls.Add(links[0].Trim()); } } } } return urls.ToArray(); } catch (Exception error) { ErrorLog.LogException(error); return new string[0]; } }
public override SiteTopic GetTopic(string url) { if (!this.User.IsLoggedIn) return null; HtmlDocument doc = new HtmlDocument(); HttpWebRequest req; HttpResult result; req = Http.Prepare(url); req.Method = "GET"; req.Referer = url; try { result = this.AllowRedirects ? Http.HandleRedirects(Http.Request(req), false) : Http.Request(req); doc.LoadHtml(result.Data); ErrorLog.LogException(result.Error); HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes("//img[@alt='Reply With Quote']"); string link = HttpUtility.HtmlDecode(nodes[0].ParentNode.GetAttributeValue("href", String.Empty)); nodes = doc.DocumentNode.SelectNodes("//span[@class='threadtitle']"); string title = HttpUtility.HtmlDecode(nodes[0].InnerText).Trim(); req = Http.Prepare((link.StartsWith("http:")) ? link : this.BaseUrl + "/" + link); req.Method = "GET"; req.Referer = url; result = this.AllowRedirects ? Http.HandleRedirects(Http.Request(req), false) : Http.Request(req); doc.LoadHtml(result.Data); ErrorLog.LogException(result.Error); string content = doc.DocumentNode.SelectNodes("//textarea[@name='message']")[0].InnerText; content = HttpUtility.HtmlDecode(content.Substring(content.IndexOf(']') + 1)).Trim(); content = content.Substring(0, content.Length - "[/quote]".Length); return new SiteTopic( title.Trim(), content.Trim(), 0, 0, url ); } catch (Exception error) { ErrorLog.LogException(error); return null; } }
public override string[] GetTopicUrls(string html) { if (html == null || html.Trim().Length == 0) return new string[0]; var urls = new List<string>(); var doc = new HtmlDocument(); HtmlNodeCollection nodes; HtmlNode p; string url; bool allowAdd = true; try { doc.LoadHtml(html); var links = from link in doc.DocumentNode.SelectNodes("//a") where link.GetAttributeValue("class", "").Contains("topictitle") select link; foreach (var a in links) { p = a.ParentNode.ParentNode; if (p.Name.ToLower() == "tr" || p.Name.ToLower() == "td") { nodes = p.SelectNodes(".//img"); foreach (var n in nodes) { if (n.GetAttributeValue("src", "").Contains("announce") || n.GetAttributeValue("src", "").Contains("sticky")) { allowAdd = false; break; } } if (allowAdd) { url = HttpUtility.HtmlDecode(a.GetAttributeValue("href", "")).TrimStart("./".ToCharArray()); urls.Add((url.StartsWith("http:")) ? url : this.BaseUrl + "/" + url); } allowAdd = true; } else if (!p.ParentNode.InnerHtml.Contains("announce_") && !p.ParentNode.InnerHtml.Contains("sticky_")) { url = HttpUtility.HtmlDecode(a.GetAttributeValue("href", "")).TrimStart("./".ToCharArray()); urls.Add((url.StartsWith("http:")) ? url : this.BaseUrl + "/" + url); } } return urls.ToArray(); } catch (Exception error) { ErrorLog.LogException(error); return new string[0]; } }
public override SiteTopic GetTopic(string url) { if (!this.User.IsLoggedIn) return null; HtmlDocument doc = new HtmlDocument(); HttpWebRequest req; HttpResult result; req = Http.Prepare(url); req.Method = "GET"; req.Referer = url; try { result = this.AllowRedirects ? Http.HandleRedirects(Http.Request(req), false) : Http.Request(req); doc.LoadHtml(result.Data); ErrorLog.LogException(result.Error); HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes("//a"); var links = (from n in nodes where HttpUtility.HtmlDecode(n.GetAttributeValue("href", "")).Contains("posting.php?mode=quote") select HttpUtility.HtmlDecode(n.GetAttributeValue("href", ""))).ToArray(); string link = links[0].TrimStart("./".ToCharArray()); req = Http.Prepare((link.StartsWith("http:")) ? link : this.BaseUrl + "/" + link); req.Method = "GET"; req.Referer = url; result = this.AllowRedirects ? Http.HandleRedirects(Http.Request(req), false) : Http.Request(req); doc.LoadHtml(result.Data); ErrorLog.LogException(result.Error); string title = doc.DocumentNode.SelectNodes("//input[@name='subject']")[0].GetAttributeValue("value", String.Empty); string content = doc.DocumentNode.SelectNodes("//textarea[@name='message']")[0].InnerText; title = HttpUtility.HtmlDecode(title); title = title.Substring(title.IndexOf(':') + 1).Trim(); content = HttpUtility.HtmlDecode(content.Substring(content.IndexOf(']') + 1)).Trim(); content = content.Substring(0, content.Length - "[/quote]".Length); return new SiteTopic( title.Trim(), content.Trim(), 0, 0, url ); } catch (Exception error) { ErrorLog.LogException(error); return null; } }
public override string[] GetTopicUrls(string html) { if (html == null || html.Trim().Length == 0) return new string[0]; var urls = new List<string>(); var doc = new HtmlDocument(); HtmlNodeCollection nodes; bool allowAdd = false; try { doc.LoadHtml(html); nodes = doc.DocumentNode.SelectNodes("//table[@class='forumline']"); var tables = (from n in nodes where (n.InnerHtml.Contains("class=\"topictitle\"") || n.InnerHtml.Contains("class='topictitle'")) && n.InnerHtml.Contains("Replies") select n).ToArray(); if (tables[tables.Length - 1].InnerHtml.Contains("<td class=\"row3\" colspan=\"6\" height=\"21\">")) { /*foreach (var n in table[table.Length - 1].SelectNodes(".//td[1]")) { if (n.InnerHtml.Contains("<b>Forum Topics</b>")) { allowAdd = false; break; } }*/ foreach (var n in tables[tables.Length - 1].SelectNodes(".//tr")) { if (!allowAdd) { nodes = n.SelectNodes(".//td[@class='row3']"); if (nodes.Count > 0 && nodes[0].InnerHtml.Contains("Topics") && !nodes[0].InnerHtml.Contains("Sticky") && nodes[0].GetAttributeValue("colspan", "") == "6") { allowAdd = true; } } else { nodes = n.SelectNodes(".//a[@class='topictitle']"); if (nodes.Count > 0) { var anchors = (from a in nodes where !a.ParentNode.InnerHtml.Contains("Announcement:</b>") && !a.ParentNode.InnerHtml.Contains("Sticky:</b>") select HttpUtility.HtmlDecode(a.GetAttributeValue("href", "")).Trim()).ToArray(); if (anchors.Length > 0 && anchors[0].Length > 0) { urls.Add((anchors[0].StartsWith("http:")) ? anchors[0] : this.BaseUrl + "/" + anchors[0]); } } } } } else { nodes = doc.DocumentNode.SelectNodes("//a[@class='topictitle']"); var anchors = from n in nodes where !n.ParentNode.InnerHtml.Contains("Announcement:</b>") && !n.ParentNode.InnerHtml.Contains("Sticky:</b>") select HttpUtility.HtmlDecode(n.GetAttributeValue("href", "")).Trim(); foreach (string a in anchors) { if (a.Length > 0) urls.Add((a.StartsWith("http:")) ? a : this.BaseUrl + "/" + a); } } return urls.ToArray(); } catch (Exception error) { ErrorLog.LogException(error); return new string[0]; } }
public override void LoginUser(string username, string password) { AsyncHelper.Run(() => { var loginPath = "/index.php?app=core&module=global§ion=login"; var html = Http.Get(this.BaseUrl + loginPath).Data; var doc = new HtmlDocument(); doc.LoadHtml(html); var auth = doc.DocumentNode.SelectSingleNode("//input[@name='auth_key']"); string url = this.BaseUrl + this.LoginPath; var details = new SiteLoginDetails(false, username, password); var data = String.Format( Res.IPBoard_314_Login, details.GetUrlSafeUsername(this.SiteEncoding), details.GetUrlSafePassword(this.SiteEncoding), auth.GetAttributeValue("value", ""), HttpUtility.UrlEncode(this.BaseUrl + loginPath, this.SiteEncoding) ); byte[] rawData = this.SiteEncoding.GetBytes(data); int check = 0; int parse = -1; this.LogoutUser(); HttpWebRequest req = Http.Prepare(url); Stream stream; req.Method = "POST"; req.Referer = url; req.ContentType = Res.FormContentType; req.ContentLength = rawData.Length; stream = req.GetRequestStream(); stream.Write(rawData, 0, rawData.Length); stream.Close(); HttpResult result = this.AllowRedirects ? Http.HandleRedirects(Http.Request(req), true) : Http.Request(req); // Did the request fail? if (result.HasError || Http.SessionCookies.Count < 2) { ErrorLog.LogException(result.Error); this.User = details; this.OnLogin(this, new LoginEventArgs(details)); return; } if (result.HasResponse) this.SiteEncoding = Encoding.GetEncoding(result.Response.CharacterSet); // Check if we did login foreach (Cookie c in Http.GetDomainCookies(req.RequestUri)) { if (c.Name.EndsWith("member_id")) { if (c.Value.Length > 0 && int.TryParse(c.Value, out parse) && parse > 0) check++; } else if (c.Name.EndsWith("pass_hash")) { if (c.Value.Length > 1) check++; } } if (check > 1) { details.IsLoggedIn = true; foreach (var c in Http.GetDomainCookies(req.RequestUri)) { details.Cookies.Add(c); } } else { var error = new Exception(String.Format( "Login check failed for '{0}'.\r\nCheck count: {1}.", this.BaseUrl, check )); ErrorLog.LogException(error); } this.User = details; this.OnLogin(this, new LoginEventArgs(details)); return; }); }