Example #1
0
 public WebScraper(ProxyCfg proxy, YtCollectCfg collectCfg, ISimpleFileStore logStore)
 {
     Proxy      = proxy;
     CollectCfg = collectCfg;
     LogStore   = logStore;
     Clients    = new ResourceCycle <HttpClient, ProxyConnectionCfg>(proxy.DirectAndProxies(), p => Task.FromResult(CreateHttpClient(p)));
 }
Example #2
0
 public YtWeb(ProxyCfg proxy, YtCollectCfg collectCfg, ISimpleFileStore logStore)
 {
     Proxy      = proxy;
     CollectCfg = collectCfg;
     LogStore   = logStore;
     Clients    = new(proxy.DirectAndProxies(), p => Task.FromResult(p.CreateHttpClient()));
 }
Example #3
0
 public ChromeScraper(ProxyCfg proxyCfg, YtCollectCfg collectCfg, ISimpleFileStore logStore)
 {
     ProxyCfg       = proxyCfg;
     CollectCfg     = collectCfg;
     LogStore       = logStore;
     ExecutablePath = new(async() => {
         var revisionInfo = await new BrowserFetcher().DownloadAsync(802497); //revision needs to be recent to be able to use optional chaining
         return(revisionInfo.ExecutablePath);
     });
 }
Example #4
0
 public ChromeScraper(ProxyCfg proxyCfg, YtCollectCfg collectCfg, ISimpleFileStore logStore)
 {
     ProxyCfg   = proxyCfg;
     CollectCfg = collectCfg;
     LogStore   = logStore;
 }