/// <summary>
/// Checks for the existence of an admin login page.
/// </summary>
/// <param name="dom">Domain to check, w/out the www.</param>
/// <param name="rule">Rule that contains the admin page value to append to the domain.
/// Rule.Value is the admin page name that is appended to the domain.
/// Rule.Property is the exact text to match on the returned admin page</param>
/// <returns></returns>
public bool Process(DOMReader dom, MarketShareRule rule)
{
try
{
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create("http://www." + dom.Domain + "/" + rule.Value);
request.UserAgent = "User-Agent Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)";
request.AllowAutoRedirect = true;
request.Method = "GET";
request.Timeout = 5000;
HttpWebResponse resp = (HttpWebResponse)request.GetResponse();
if (resp.StatusCode == HttpStatusCode.OK)
{
Regex r = new Regex(rule.Property, RegexOptions.IgnoreCase);
HtmlAgilityPack.HtmlNode source = dom.Document.DocumentNode.SelectSingleNode("html");
if (!Object.Equals(null, source))
{
return(r.Match(source.InnerHtml).Success);
}
}
return(false);
}
catch (Exception)
{
//todo: log
}
return(false);
}
/// <summary>
/// Find rule value in a collection of tag URLs
/// </summary>
/// <param name="tags"></param>
/// <param name="rule"></param>
/// <returns></returns>
public bool ExistsInCollection(HtmlNodeCollection tags, MarketShareRule rule)
{
foreach (var item in tags)
{
if (item.Attributes.Contains(rule.Property))
{
string propValue = item.Attributes[rule.Property].Value.ToLower();
if (propValue.Length >= rule.Value.Length && propValue.Contains(rule.Value.ToLower()))
{
if (!rule.IgnoreAbsURI && IsAbsoluteUrl(propValue))
{
if (propValue.Contains(this.Domain))
{
return(true);
}
}
else
{
return(true);
}
}
}
}
return(false);
}
/// <summary>
/// Gets the cookies for the domain and determines if there is a rule match
/// </summary>
/// <param name="dom">The DocumentReader object for the domain</param>
/// <param name="rule">The rule to match</param>
/// <returns>True if a cookie is found matching the rule</returns>
public bool Process(DOMReader dom, MarketShareRule rule)
{
try
{
if (Object.Equals(null, dom.RequestCookies) && Object.Equals(null, dom.ResponseCookies))
{
//get cookies and look for a match
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create("http://www." + dom.Domain);
CookieContainer cookieJar = new CookieContainer();
request.UserAgent = "User-Agent Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)";
request.CookieContainer = cookieJar;
request.AllowAutoRedirect = true;
request.Method = "GET";
request.Timeout = 5000;
HttpWebResponse resp = (HttpWebResponse)request.GetResponse();
dom.RequestCookies = cookieJar.GetCookies(request.RequestUri);
dom.ResponseCookies = cookieJar.GetCookies(resp.ResponseUri);
}
if (dom.RequestCookies != null)
{
foreach (Cookie c1 in dom.RequestCookies)
{
if (c1.Name.Contains(rule.Value))
{
return(true);
}
}
}
if (dom.ResponseCookies != null)
{
foreach (Cookie c in dom.ResponseCookies)
{
if (c.Name.Contains(rule.Value))
{
return(true);
}
}
}
}
catch (Exception e)
{
if (!e.Message.Contains("404") && !e.Message.Contains("timed out"))
{
//todo: log
//Utility.WriteToLogFile(String.Format("SmallBiz_NoCookieInfo_{0:M_d_yyyy}", DateTime.Today) + ".log", string.Format("Domain: {0}", dom.Domain));
}
}
return(false);
}
private bool ValidateRule(MarketShareRule rule, DOMReader dom)
{
//default to text, so that it will just regex the whole page
IProcessor proc = _processors["text"];
if (_processors.ContainsKey(rule.Type.ToLower()))
{
proc = _processors[rule.Type.ToLower()];
}
return(proc.Process(dom, rule));
}
/// <summary>
/// Determines if the value exists in the Meta tag collection of the document
/// </summary>
/// <param name="dom">DocumentReader object for the domain</param>
/// <param name="rule">The rule to match</param>
/// <returns>True if the rule value is found in the meta tag collection.</returns>
public bool Process(DOMReader dom, MarketShareRule rule)
{
try
{
return(dom.ExistsInCollection(dom.MetaTags, rule));
}
catch (Exception e)
{
ExceptionExtensions.LogWarning(e, "API.MarketAnalysis.Metas.Process()", string.Format("Domain: {0}, {1}", dom.Domain, rule.ToString()));
}
return(false);
}
public bool Process(DOMReader dom, MarketShareRule rule)
{
try
{
return(dom.Domain.Contains(rule.Value));
}
catch (Exception e)
{
ExceptionExtensions.LogWarning(e, "API.MarketAnalysis.Urls.Process()", string.Format("Domain: {0}, {1}", dom.Domain, rule.ToString()));
}
return(false);
}
/// <summary>
/// Uses RegEx to determines if the value exists in the text of the document source.
/// </summary>
/// <param name="dom">DocumentReader object for the domain</param>
/// <param name="rule">The rule to match</param>
/// <returns>True if the rule value is found in the source of the document.</returns>
public bool Process(DOMReader dom, MarketShareRule rule)
{
try
{
Regex r = new Regex(rule.Value.Replace("{domain}", "www." + dom.Domain), RegexOptions.IgnoreCase);
HtmlAgilityPack.HtmlNode source = dom.Document.DocumentNode.SelectSingleNode(rule.Property);
if (!Object.Equals(null, source))
{
return(r.Match(source.InnerHtml).Success);
}
}
catch (Exception e)
{
ExceptionExtensions.LogWarning(e, "API.MarketAnalysis.Text.Process()", string.Format("Domain: {0}, {1}", dom.Domain, rule.ToString()));
}
return(false);
}
/// <summary>
/// Determines if the value exists in the Stylesheet tag collection of the document
/// </summary>
/// <param name="dom">DocumentReader object for the domain</param>
/// <param name="rule">The rule to match</param>
/// <returns>True if the rule value is found in the stylesheet tag collection.</returns>
public bool Process(DOMReader dom, MarketShareRule rule)
{
try
{
foreach (var item in dom.Stylesheets)
{
if (item.Attributes.Contains(rule.Property))
{
string propValue = item.Attributes[rule.Property].Value.ToLower();
return(propValue.Contains(rule.Value.ToLower()));
}
}
return(false);
}
catch (Exception e)
{
ExceptionExtensions.LogWarning(e, "API.MarketAnalysis.Stylesheets.Process()", string.Format("Domain: {0}, {1}", dom.Domain, rule.ToString()));
}
return(false);
}
/// <summary>
/// Determines if the Generator tag contains the value in the rule
/// </summary>
/// <param name="dom">DocumentReader object for the domain</param>
/// <param name="rule">The rule to match</param>
/// <returns>True if the rule value is found in the generator tag collection</returns>
public bool Process(DOMReader dom, MarketShareRule rule)
{
try
{
foreach (HtmlAgilityPack.HtmlNode node in dom.GeneratorTags)
{
if (!Object.Equals(null, node.Attributes[rule.Property]))
{
if (node.Attributes[rule.Property].Value.ToLower().Contains(rule.Value.ToLower()))
{
return(true);
}
}
}
}
catch (Exception e)
{
ExceptionExtensions.LogWarning(e, "API.MarketAnalysis.Generators.Process()", string.Format("Domain: {0}, {1}", dom.Domain, rule.ToString()));
}
return(false);
}