public static List <Uri> AsListOfProxies(this IBrowsingResponse response)
{
var text = response.AsFixedXML().ToCleanText();
foreach (var i in Replacements)
{
text = text.Replace(i.Key, i.Value);
}
var ret = new List <Uri>();
var matchesIp = proxyIpRegex.Matches(text);
foreach (Match m in matchesIp)
{
var ip = m.Groups[1].Value;
var port = String.IsNullOrEmpty(m.Groups[7].Value) ?
"3128" : m.Groups[7].Value;
var uriStr = String.Format(@"http://{0}:{1}", ip, port);
ret.Add(new Uri(uriStr));
}
/* - TODO Proxy resolver by name
* var matchesName = proxyNameRegex.Matches(text);
* foreach (Match m in matchesName)
* {
* var ip = m.Groups[1].Value;
* var port = String.IsNullOrEmpty(m.Groups[7].Value) ?
* "3128" : m.Groups[3].Value;
* var uriStr = String.Format(@"http://{0}:{1}", ip, port);
* ret.Add(new Uri(uriStr));
* }
*/
return(ret);
}
public static FlowAssertion IsStatusNotOK(this ScriptFlow flow, IBrowsingResponse response)
{
var message =
String.Format("Response from '{0}' not ok ({1})", response.ResponseUrl, response.StatusCode);
return flow.IsStatusOK(response).Otherwise().Warn(message);
}
public static FlowAssertion IsStatusNotOK(this ScriptFlow flow, IBrowsingResponse response)
{
var message =
String.Format("Response from '{0}' not ok ({1})", response.ResponseUrl, response.StatusCode);
return(flow.IsStatusOK(response).Otherwise().Warn(message));
}
public IBrowsingResponse NavigateFile(Uri httpUrl, List <Tuple <string, string> > filePaths, NameValueCollection postParamz)
{
IBrowsingResponse response = null;
List <Tuple <string, string, Stream> > files = new List <Tuple <string, string, Stream> >();
try
{
filePaths.ForEach((filepath) =>
{
var file = File.OpenRead(filepath.Item2);
files.Add(
new Tuple <string, string, Stream>(filepath.Item1, Path.GetFileName(filepath.Item2), file));
});
response = NavigateFile(httpUrl, files, postParamz);
}
catch (Exception ex)
{
throw new Exception("Upload failed", ex);
}
finally
{
foreach (var tuple in files)
{
tuple.Item3.Close();
}
}
return(response);
}
public static string FixPathToAbsolute(this IBrowsingResponse response, string link)
{
Uri uri = null;
try
{
if (Uri.TryCreate(link, UriKind.Absolute, out uri))
{
return(uri.ToString());
}
}
catch (Exception)
{
}
var ret = "";
//is relative
if (link.StartsWith("/"))
{
ret = string.Format("{0}://{1}{2}", response.ResponseUrl.Scheme,
response.ResponseUrl.Host, link);
}
else
{
var url = response.ResponseUrl.ToString();
url = url.Substring(0, url.LastIndexOf("/"));
ret = string.Format("{0}/{1}", url, link);
}
if (ret.IndexOf('#') > 0)
{
ret = ret.Substring(0, ret.IndexOf('#'));
}
return(ret);
}
public static IEnumerable <string> ExtractHyperlinks(this IBrowsingResponse response)
{
var doc = response.AsHtmlDocument();
var ret = ExtractAnchorLinks(doc);
return(response.FixPathToAbsolute(ret));
}
private bool ShouldCache(IBrowsingResponse response)
{
switch (StateCaching)
{
case StateCaching.All:
return(true);
case StateCaching.OnlyValid:
return(!(response is ErrorBrowsingResponse));
case StateCaching.Never:
return(false);
case StateCaching.LikeBrowser:
var doCache = !(response is ErrorBrowsingResponse);
if (doCache && !String.IsNullOrEmpty(response.Headers["Cache-Control"]))
{
doCache = !(response.Headers["Cache-Control"].Contains("no-cache") ||
response.Headers["Cache-Control"].Contains("no-store") ||
response.Headers["Cache-Control"].Contains("max-age=0,") ||
response.Headers["Cache-Control"].Contains("must-revalidate"));
}
if (doCache && !String.IsNullOrEmpty(response.Headers["Pragma"]))
{
doCache = !(response.Headers["Pragma"].Contains("no-cache"));
}
return(doCache);
default:
throw new InvalidOperationException("Unknown StateCaching: " + StateCaching);
}
}
public static IEnumerable <string> ExtractAlllinks(this IBrowsingResponse response)
{
var doc = response.AsHtmlDocument();
var ret = doc.ExtractAnchorLinks().ToList();
ret.AddRange(doc.ExtractLinksNonAnchorLinks());
return(response.FixPathToAbsolute(ret));
}
public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
{
if (retries < Attempts&&response.StatusCode == HttpStatusCode.ServiceUnavailable)
{
retries++;
session.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(session, Descriminator));
return true;
}
return base.OnAfterRequestRerun(session, uri, response);
}
public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
{
if (retries < Attempts&&response.StatusCode == HttpStatusCode.ServiceUnavailable)
{
retries++;
Thread.Sleep(SleepInterval);
return true;
}
return base.OnAfterRequestRerun(session, uri, response);
}
public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
{
if (retries < Attempts && response.StatusCode == HttpStatusCode.ServiceUnavailable)
{
retries++;
session.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(session, Descriminator));
return(true);
}
return(base.OnAfterRequestRerun(session, uri, response));
}
public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
{
if (retries < Attempts && response.StatusCode == HttpStatusCode.ServiceUnavailable)
{
retries++;
Thread.Sleep(SleepInterval);
return(true);
}
return(base.OnAfterRequestRerun(session, uri, response));
}
public static XDocument AsFixedXML(this IBrowsingResponse response)
{
try
{
return(AsFixedXML(response.ResponseContent));
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine(ex);
var doc = AsFixedXML(HtmlDocument.HtmlEncode(response.ResponseContent));
return(doc);
}
}
/// <summary>
/// Pretti buggy approach
/// TODO: make a better approach
/// </summary>
/// <param name="response"></param>
/// <returns></returns>
public static string ContentWithFixedToAbsoluteLinks(this IBrowsingResponse response)
{
var linksR = ExtractAlllinks(response).ToArray();
var linksA = FixPathToAbsolute(response, linksR).ToArray();
var result = response.ResponseContent + "";
for (int i = 0; i < linksR.Length; i++)
{
var linkR = "\"" + linksR[i] + "\"";
var linkA = "\"" + linksA[i] + "\"";
result = result.Replace(linkR, linkA);
}
return(result);
}
public IBrowsingResponse NavigatePost(Uri httpUrl, NameValueCollection postParamz)
{
if (decorators.FindAll(x => x.IsEnabled).Select(x => x.OnBeforeRequestStop(this, httpUrl))
.Any(x => x))
{
return(null);
}
IBrowsingResponse browsingResponse = null;
do
{
browsingResponse = CurrentProxy.PostResponse(httpUrl, postParamz, this);
} while (
decorators.FindAll(x => x.IsEnabled).Select(x => x.OnAfterRequestRerun(this, httpUrl, browsingResponse))
.Any(x => x));
return(browsingResponse);
}
public static void SaveToFile(this IBrowsingResponse response, string filename)
{
var file = File.Open(filename, FileMode.Create);
var responseStream = response.ResponseStream;
Byte[] buffer = new Byte[4096];
var startPoint = 0;
var endLoad = false;
do
{
int bytesSize = responseStream.Read(buffer, 0, buffer.Length);
if (bytesSize > 0)
{
file.Write(buffer, 0, bytesSize);
startPoint += bytesSize;
}
else
{
endLoad = true;
}
} while (!endLoad);
file.Close();
}
public virtual bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
{
return(false);
}
public static FlowAssertion IsCached(this ScriptFlow flow, IBrowsingResponse resp)
{
return FlowAssertion.Create(flow, (resp is StateBrowsingResponse));
}
public static IEnumerable <HtmlNode> ExtractInputFields(this IBrowsingResponse response)
{
var doc = response.AsHtmlDocument();
return(ExtractInputFields(doc));
}
public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
{
LastRequest = uri;
return(base.OnAfterRequestRerun(session, uri, response));
}
public virtual bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
{
return false;
}
public static StateBrowsingResponse Create(IBrowsingResponse resp)
{
var map = ObjectMapperManager.DefaultInstance.GetMapper <IBrowsingResponse, StateBrowsingResponse>().Map(resp);
return(map);
}
public static FlowAssertion IsStatusOK(this ScriptFlow flow, IBrowsingResponse response)
{
return(FlowAssertion
.Create(flow, response.StatusCode == System.Net.HttpStatusCode.OK));
}
public static FlowAssertion IsCached(this ScriptFlow flow, IBrowsingResponse resp)
{
return(FlowAssertion.Create(flow, (resp is StateBrowsingResponse)));
}
public static StateBrowsingResponse Create(IBrowsingResponse resp)
{
var map = ObjectMapperManager.DefaultInstance.GetMapper<IBrowsingResponse, StateBrowsingResponse>().Map(resp);
return map;
}
/// <summary>
/// Add domain to partial links
/// </summary>
/// <param name="response"></param>
/// <param name="linkList"></param>
/// <returns></returns>
public static IEnumerable <string> FixPathToAbsolute(this IBrowsingResponse response, IEnumerable <string> linkList)
{
return(linkList.Select(x => response.FixPathToAbsolute(x)));
}
public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
{
LastRequest = uri;
return base.OnAfterRequestRerun(session, uri, response);
}
/// <summary>
/// http://htmlagilitypack.codeplex.com/wikipage?title=Examples
/// http://www.w3schools.com/xpath/xpath_syntax.asp
/// </summary>
/// <param name="response"></param>
/// <returns></returns>
public static HtmlDocument AsHtmlDocument(this IBrowsingResponse response)
{
return(AsHtmlDocument(response.ResponseContent));
}
public static XDocument AsXML(this IBrowsingResponse response)
{
return(XDocument.Parse(response.ResponseContent));
}
/// <summary>
/// http://james.newtonking.com/projects/json/help/
/// </summary>
/// <param name="response"></param>
/// <returns></returns>
public static JObject AsJSON(this IBrowsingResponse response)
{
return(JObject.Parse(response.ResponseContent));
}
public static FlowAssertion IsStatusOK(this ScriptFlow flow, IBrowsingResponse response)
{
return FlowAssertion
.Create(flow, response.StatusCode == System.Net.HttpStatusCode.OK);
}
