public ProxyHttpRequestSelector(ProxySelector selector, IHttpWire wire) : base(wire) { _selector = selector; _selectorCopy = new HttpProxySelector(selector.FetchProxies()); _error = new Stack<AttemptPair>(); }
public ProxyHttpRequestSelector(ProxySelector selector, IHttpWire wire) : base(wire) { _selector = selector; _selectorCopy = new HttpProxySelector(selector.FetchProxies()); _error = new Stack <AttemptPair>(); }
private static DownloadImage GetImage(IHttpRequestFactory factory, IHttpWire wire) { var request = CreateRequest(factory, wire); var bytes = request.Download(); string extension = Path.GetExtension(wire.Url); string fileName = Guid.NewGuid().ToString("N") + extension; if (bytes.Length == 0) fileName = ""; return new DownloadImage() { date = DateTime.Now, image = bytes, size = bytes.Length, url = wire.Url, filename = fileName }; }
private static RuntimeTable<DownloadPage> DownloadPage(IRuntime runtime, IHttpWire[] wires) { var table = new RuntimeTable<DownloadPage>(); foreach (var wire in wires) { int contentlength; var doc = GetDocument(runtime.RequestFactory, wire, out contentlength); table.Add(new DownloadPage() { url = wire.Url, nodes = new[] { doc.DocumentNode }, date = DateTime.Now, size = contentlength }); } return table; }
private static HtmlDocument GetDocument(IHttpRequestFactory factory, IHttpWire wire, out int length) { var request = CreateRequest(factory, wire); var bytes = request.Download(); string html = string.Empty; length = bytes.Length; html = Encoding.UTF8.GetString(bytes, 0, bytes.Length); HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(html); return doc; }
private static HtmlDocument GetDocument(IHttpRequestFactory factory, IHttpWire wire, out int length) { var request = CreateRequest(factory, wire); var bytes = request.Download(); string html = string.Empty; length = bytes.Length; html = Encoding.UTF8.GetString(bytes, 0, bytes.Length); HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(html); return(doc); }
private void Work(string logValue) //multi threaded { ThreadContext.Properties[Config.LogKey] = logValue; IHttpWire wire = null; while (true) { lock (UrlLock) { if (Wires.Count > 0) { wire = Wires.Dequeue(); } else { wire = null; } } if (wire == null) //nothing left in queue { break; } var downloadResult = Fetch(_args.Runtime, wire); if (_callOnProgres) { _args.Runtime.OnProgress(); } lock (ResultLock) { foreach (var p in downloadResult) { _results.Enqueue(p); lock (ResultCountLock) { _resultCount++; } } } } FinishedDownloading = true; }
private static DownloadImage GetImage(IHttpRequestFactory factory, IHttpWire wire) { var request = CreateRequest(factory, wire); var bytes = request.Download(); string extension = Path.GetExtension(wire.Url); string fileName = Guid.NewGuid().ToString("N") + extension; if (bytes.Length == 0) { fileName = ""; } return(new DownloadImage() { date = DateTime.Now, image = bytes, size = bytes.Length, url = wire.Url, filename = fileName }); }
private static HtmlDoc GetDocument(IHttpRequestFactory factory, IHttpWire wire, out int length) { var request = CreateRequest(factory, wire); var bytes = request.Download() as byte[]; if (bytes == null) { bytes = new byte[0]; } string html = string.Empty; length = bytes.Length; html = Encoding.UTF8.GetString(bytes, 0, bytes.Length); HtmlDoc doc = Config.DomFactory.Create(); doc.Load(html); return(doc); }
private static RuntimeTable <DynamicObject> ProcesJson(IHttpRequestFactory factory, IHttpWire wire) { var request = CreateRequest(factory, wire); var json = request.Download() as string; if (json == null) { json = string.Empty; } dynamic serializedValue = JsonConvert.DeserializeObject(json); var dynamics = new List <DynamicObject>(); if (serializedValue != null && serializedValue is IEnumerable <dynamic> ) { IEnumerable <dynamic> jsonArray = serializedValue; if (!(serializedValue is JArray)) { jsonArray = new[] { serializedValue } } ; var properties = new List <Dictionary <string, object> >(); foreach (dynamic v in jsonArray) { var dynamic = new DynamicObject(); Dictionary <string, object> values = v.ToObject <Dictionary <string, object> >(); foreach (var p in values.Keys) { dynamic.Add(p, values[p].ToString()); } dynamics.Add(dynamic); } } var table = new RuntimeTable <DynamicObject>(); table.SetRows(dynamics); return(table); }
public static RuntimeTable <DownloadPage> DownloadPage(IRuntime runtime, IHttpWire wire) { return(DownloadPage(runtime, new IHttpWire[] { wire })); }
private static IHttpRequest CreateRequest(IHttpRequestFactory factory, IHttpWire wire) { return(factory.Create(wire)); }
public static RuntimeTable<DownloadPage> DownloadPage(IRuntime runtime, IHttpWire wire) { return DownloadPage(runtime, new IHttpWire[] { wire }); }
public static RuntimeTable <DynamicObject> DownloadJSPage(IRuntime runtime, IHttpWire wire) { return(ProcesJson(runtime.RequestFactory, wire)); }
public override IHttpRequest Create(IHttpWire wire) { return(new RetryHttpRequest(wire)); }
protected abstract RuntimeTable <TRow> Fetch(IRuntime runtime, IHttpWire wire);
protected override RuntimeTable <DynamicObject> Fetch(IRuntime runtime, IHttpWire wire) { return(Http.DownloadJSPage(runtime, wire)); }
public ProxyHttpRequest(Proxy proxy, IHttpWire wire) : base(wire) { _proxy = proxy; }
public override IHttpRequest Create(IHttpWire wire) { return new ProxyHttpRequestSelector(_selector, wire); }
public override IHttpRequest Create(IHttpWire wire) { return(new ProxyHttpRequestSelector(_selector, wire)); }
public override IHttpRequest Create(IHttpWire url) { return(new ProxyHttpRequest(_proxy, url)); }
public abstract IHttpRequest Create(IHttpWire url);
public RetryHttpRequest(IHttpWire wire) : base(wire) { _errorCount = 0; }
protected override RuntimeTable <DownloadPage> Fetch(IRuntime runtime, IHttpWire wire) { return(Http.DownloadPage(runtime, wire)); }
public ProxyHttpRequest(Proxy proxy, IHttpWire wire) : base(wire) { _proxy = proxy; }
public abstract IHttpRequest Create(IHttpWire url);
private static IHttpRequest CreateRequest(IHttpRequestFactory factory, IHttpWire wire) { return factory.Create(wire); }
protected HttpRequest(IHttpWire wire) { Wire = wire as HttpWire; }
public override IHttpRequest Create(IHttpWire wire) { return new RetryHttpRequest(wire); }
protected HttpRequest(IHttpWire wire) { Wire = wire; }
public override IHttpRequest Create(IHttpWire url) { return new ProxyHttpRequest(_proxy, url); }