public ProxyHttpRequestSelector(ProxySelector selector, IHttpWire wire)
: base(wire)
{
_selector = selector;
_selectorCopy = new HttpProxySelector(selector.FetchProxies());
_error = new Stack<AttemptPair>();
}
public ProxyHttpRequestSelector(ProxySelector selector, IHttpWire wire)
: base(wire)
{
_selector = selector;
_selectorCopy = new HttpProxySelector(selector.FetchProxies());
_error = new Stack <AttemptPair>();
}
private static DownloadImage GetImage(IHttpRequestFactory factory, IHttpWire wire)
{
var request = CreateRequest(factory, wire);
var bytes = request.Download();
string extension = Path.GetExtension(wire.Url);
string fileName = Guid.NewGuid().ToString("N") + extension;
if (bytes.Length == 0)
fileName = "";
return new DownloadImage() { date = DateTime.Now, image = bytes, size = bytes.Length, url = wire.Url, filename = fileName };
}
private static RuntimeTable<DownloadPage> DownloadPage(IRuntime runtime, IHttpWire[] wires)
{
var table = new RuntimeTable<DownloadPage>();
foreach (var wire in wires)
{
int contentlength;
var doc = GetDocument(runtime.RequestFactory, wire, out contentlength);
table.Add(new DownloadPage() { url = wire.Url, nodes = new[] { doc.DocumentNode }, date = DateTime.Now, size = contentlength });
}
return table;
}
private static HtmlDocument GetDocument(IHttpRequestFactory factory, IHttpWire wire, out int length)
{
var request = CreateRequest(factory, wire);
var bytes = request.Download();
string html = string.Empty;
length = bytes.Length;
html = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(html);
return doc;
}
private static HtmlDocument GetDocument(IHttpRequestFactory factory, IHttpWire wire, out int length)
{
var request = CreateRequest(factory, wire);
var bytes = request.Download();
string html = string.Empty;
length = bytes.Length;
html = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(html);
return(doc);
}
private void Work(string logValue) //multi threaded
{
ThreadContext.Properties[Config.LogKey] = logValue;
IHttpWire wire = null;
while (true)
{
lock (UrlLock)
{
if (Wires.Count > 0)
{
wire = Wires.Dequeue();
}
else
{
wire = null;
}
}
if (wire == null) //nothing left in queue
{
break;
}
var downloadResult = Fetch(_args.Runtime, wire);
if (_callOnProgres)
{
_args.Runtime.OnProgress();
}
lock (ResultLock)
{
foreach (var p in downloadResult)
{
_results.Enqueue(p);
lock (ResultCountLock)
{
_resultCount++;
}
}
}
}
FinishedDownloading = true;
}
private static DownloadImage GetImage(IHttpRequestFactory factory, IHttpWire wire)
{
var request = CreateRequest(factory, wire);
var bytes = request.Download();
string extension = Path.GetExtension(wire.Url);
string fileName = Guid.NewGuid().ToString("N") + extension;
if (bytes.Length == 0)
{
fileName = "";
}
return(new DownloadImage()
{
date = DateTime.Now, image = bytes, size = bytes.Length, url = wire.Url, filename = fileName
});
}
private static HtmlDoc GetDocument(IHttpRequestFactory factory, IHttpWire wire, out int length)
{
var request = CreateRequest(factory, wire);
var bytes = request.Download() as byte[];
if (bytes == null)
{
bytes = new byte[0];
}
string html = string.Empty;
length = bytes.Length;
html = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
HtmlDoc doc = Config.DomFactory.Create();
doc.Load(html);
return(doc);
}
private static RuntimeTable <DynamicObject> ProcesJson(IHttpRequestFactory factory, IHttpWire wire)
{
var request = CreateRequest(factory, wire);
var json = request.Download() as string;
if (json == null)
{
json = string.Empty;
}
dynamic serializedValue = JsonConvert.DeserializeObject(json);
var dynamics = new List <DynamicObject>();
if (serializedValue != null && serializedValue is IEnumerable <dynamic> )
{
IEnumerable <dynamic> jsonArray = serializedValue;
if (!(serializedValue is JArray))
{
jsonArray = new[] { serializedValue }
}
;
var properties = new List <Dictionary <string, object> >();
foreach (dynamic v in jsonArray)
{
var dynamic = new DynamicObject();
Dictionary <string, object> values = v.ToObject <Dictionary <string, object> >();
foreach (var p in values.Keys)
{
dynamic.Add(p, values[p].ToString());
}
dynamics.Add(dynamic);
}
}
var table = new RuntimeTable <DynamicObject>();
table.SetRows(dynamics);
return(table);
}
public static RuntimeTable <DownloadPage> DownloadPage(IRuntime runtime, IHttpWire wire)
{
return(DownloadPage(runtime, new IHttpWire[] { wire }));
}
private static IHttpRequest CreateRequest(IHttpRequestFactory factory, IHttpWire wire)
{
return(factory.Create(wire));
}
public static RuntimeTable<DownloadPage> DownloadPage(IRuntime runtime, IHttpWire wire)
{
return DownloadPage(runtime, new IHttpWire[] { wire });
}
public static RuntimeTable <DynamicObject> DownloadJSPage(IRuntime runtime, IHttpWire wire)
{
return(ProcesJson(runtime.RequestFactory, wire));
}
public override IHttpRequest Create(IHttpWire wire)
{
return(new RetryHttpRequest(wire));
}
protected abstract RuntimeTable <TRow> Fetch(IRuntime runtime, IHttpWire wire);
protected override RuntimeTable <DynamicObject> Fetch(IRuntime runtime, IHttpWire wire)
{
return(Http.DownloadJSPage(runtime, wire));
}
public ProxyHttpRequest(Proxy proxy, IHttpWire wire)
: base(wire)
{
_proxy = proxy;
}
public override IHttpRequest Create(IHttpWire wire)
{
return new ProxyHttpRequestSelector(_selector, wire);
}
public override IHttpRequest Create(IHttpWire wire)
{
return(new ProxyHttpRequestSelector(_selector, wire));
}
public override IHttpRequest Create(IHttpWire url)
{
return(new ProxyHttpRequest(_proxy, url));
}
public abstract IHttpRequest Create(IHttpWire url);
public RetryHttpRequest(IHttpWire wire)
: base(wire)
{
_errorCount = 0;
}
protected override RuntimeTable <DownloadPage> Fetch(IRuntime runtime, IHttpWire wire)
{
return(Http.DownloadPage(runtime, wire));
}
public ProxyHttpRequest(Proxy proxy, IHttpWire wire)
: base(wire)
{
_proxy = proxy;
}
public abstract IHttpRequest Create(IHttpWire url);
private static IHttpRequest CreateRequest(IHttpRequestFactory factory, IHttpWire wire)
{
return factory.Create(wire);
}
protected HttpRequest(IHttpWire wire)
{
Wire = wire as HttpWire;
}
public override IHttpRequest Create(IHttpWire wire)
{
return new RetryHttpRequest(wire);
}
protected HttpRequest(IHttpWire wire)
{
Wire = wire;
}
public override IHttpRequest Create(IHttpWire url)
{
return new ProxyHttpRequest(_proxy, url);
}
