/// <summary> /// Checks for the existence of an admin login page. /// </summary> /// <param name="dom">Domain to check, w/out the www.</param> /// <param name="rule">Rule that contains the admin page value to append to the domain. /// Rule.Value is the admin page name that is appended to the domain. /// Rule.Property is the exact text to match on the returned admin page</param> /// <returns></returns> public bool Process(DOMReader dom, MarketShareRule rule) { try { HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create("http://www." + dom.Domain + "/" + rule.Value); request.UserAgent = "User-Agent Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"; request.AllowAutoRedirect = true; request.Method = "GET"; request.Timeout = 5000; HttpWebResponse resp = (HttpWebResponse)request.GetResponse(); if (resp.StatusCode == HttpStatusCode.OK) { Regex r = new Regex(rule.Property, RegexOptions.IgnoreCase); HtmlAgilityPack.HtmlNode source = dom.Document.DocumentNode.SelectSingleNode("html"); if (!Object.Equals(null, source)) { return(r.Match(source.InnerHtml).Success); } } return(false); } catch (Exception) { //todo: log } return(false); }
/// <summary> /// Find rule value in a collection of tag URLs /// </summary> /// <param name="tags"></param> /// <param name="rule"></param> /// <returns></returns> public bool ExistsInCollection(HtmlNodeCollection tags, MarketShareRule rule) { foreach (var item in tags) { if (item.Attributes.Contains(rule.Property)) { string propValue = item.Attributes[rule.Property].Value.ToLower(); if (propValue.Length >= rule.Value.Length && propValue.Contains(rule.Value.ToLower())) { if (!rule.IgnoreAbsURI && IsAbsoluteUrl(propValue)) { if (propValue.Contains(this.Domain)) { return(true); } } else { return(true); } } } } return(false); }
/// <summary> /// Gets the cookies for the domain and determines if there is a rule match /// </summary> /// <param name="dom">The DocumentReader object for the domain</param> /// <param name="rule">The rule to match</param> /// <returns>True if a cookie is found matching the rule</returns> public bool Process(DOMReader dom, MarketShareRule rule) { try { if (Object.Equals(null, dom.RequestCookies) && Object.Equals(null, dom.ResponseCookies)) { //get cookies and look for a match HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create("http://www." + dom.Domain); CookieContainer cookieJar = new CookieContainer(); request.UserAgent = "User-Agent Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"; request.CookieContainer = cookieJar; request.AllowAutoRedirect = true; request.Method = "GET"; request.Timeout = 5000; HttpWebResponse resp = (HttpWebResponse)request.GetResponse(); dom.RequestCookies = cookieJar.GetCookies(request.RequestUri); dom.ResponseCookies = cookieJar.GetCookies(resp.ResponseUri); } if (dom.RequestCookies != null) { foreach (Cookie c1 in dom.RequestCookies) { if (c1.Name.Contains(rule.Value)) { return(true); } } } if (dom.ResponseCookies != null) { foreach (Cookie c in dom.ResponseCookies) { if (c.Name.Contains(rule.Value)) { return(true); } } } } catch (Exception e) { if (!e.Message.Contains("404") && !e.Message.Contains("timed out")) { //todo: log //Utility.WriteToLogFile(String.Format("SmallBiz_NoCookieInfo_{0:M_d_yyyy}", DateTime.Today) + ".log", string.Format("Domain: {0}", dom.Domain)); } } return(false); }
private bool ValidateRule(MarketShareRule rule, DOMReader dom) { //default to text, so that it will just regex the whole page IProcessor proc = _processors["text"]; if (_processors.ContainsKey(rule.Type.ToLower())) { proc = _processors[rule.Type.ToLower()]; } return(proc.Process(dom, rule)); }
/// <summary> /// Determines if the value exists in the Meta tag collection of the document /// </summary> /// <param name="dom">DocumentReader object for the domain</param> /// <param name="rule">The rule to match</param> /// <returns>True if the rule value is found in the meta tag collection.</returns> public bool Process(DOMReader dom, MarketShareRule rule) { try { return(dom.ExistsInCollection(dom.MetaTags, rule)); } catch (Exception e) { ExceptionExtensions.LogWarning(e, "API.MarketAnalysis.Metas.Process()", string.Format("Domain: {0}, {1}", dom.Domain, rule.ToString())); } return(false); }
public bool Process(DOMReader dom, MarketShareRule rule) { try { return(dom.Domain.Contains(rule.Value)); } catch (Exception e) { ExceptionExtensions.LogWarning(e, "API.MarketAnalysis.Urls.Process()", string.Format("Domain: {0}, {1}", dom.Domain, rule.ToString())); } return(false); }
/// <summary> /// Uses RegEx to determines if the value exists in the text of the document source. /// </summary> /// <param name="dom">DocumentReader object for the domain</param> /// <param name="rule">The rule to match</param> /// <returns>True if the rule value is found in the source of the document.</returns> public bool Process(DOMReader dom, MarketShareRule rule) { try { Regex r = new Regex(rule.Value.Replace("{domain}", "www." + dom.Domain), RegexOptions.IgnoreCase); HtmlAgilityPack.HtmlNode source = dom.Document.DocumentNode.SelectSingleNode(rule.Property); if (!Object.Equals(null, source)) { return(r.Match(source.InnerHtml).Success); } } catch (Exception e) { ExceptionExtensions.LogWarning(e, "API.MarketAnalysis.Text.Process()", string.Format("Domain: {0}, {1}", dom.Domain, rule.ToString())); } return(false); }
/// <summary> /// Determines if the value exists in the Stylesheet tag collection of the document /// </summary> /// <param name="dom">DocumentReader object for the domain</param> /// <param name="rule">The rule to match</param> /// <returns>True if the rule value is found in the stylesheet tag collection.</returns> public bool Process(DOMReader dom, MarketShareRule rule) { try { foreach (var item in dom.Stylesheets) { if (item.Attributes.Contains(rule.Property)) { string propValue = item.Attributes[rule.Property].Value.ToLower(); return(propValue.Contains(rule.Value.ToLower())); } } return(false); } catch (Exception e) { ExceptionExtensions.LogWarning(e, "API.MarketAnalysis.Stylesheets.Process()", string.Format("Domain: {0}, {1}", dom.Domain, rule.ToString())); } return(false); }
/// <summary> /// Determines if the Generator tag contains the value in the rule /// </summary> /// <param name="dom">DocumentReader object for the domain</param> /// <param name="rule">The rule to match</param> /// <returns>True if the rule value is found in the generator tag collection</returns> public bool Process(DOMReader dom, MarketShareRule rule) { try { foreach (HtmlAgilityPack.HtmlNode node in dom.GeneratorTags) { if (!Object.Equals(null, node.Attributes[rule.Property])) { if (node.Attributes[rule.Property].Value.ToLower().Contains(rule.Value.ToLower())) { return(true); } } } } catch (Exception e) { ExceptionExtensions.LogWarning(e, "API.MarketAnalysis.Generators.Process()", string.Format("Domain: {0}, {1}", dom.Domain, rule.ToString())); } return(false); }