public WebScraper(ProxyCfg proxy, YtCollectCfg collectCfg, ISimpleFileStore logStore) { Proxy = proxy; CollectCfg = collectCfg; LogStore = logStore; Clients = new ResourceCycle <HttpClient, ProxyConnectionCfg>(proxy.DirectAndProxies(), p => Task.FromResult(CreateHttpClient(p))); }
public YtWeb(ProxyCfg proxy, YtCollectCfg collectCfg, ISimpleFileStore logStore) { Proxy = proxy; CollectCfg = collectCfg; LogStore = logStore; Clients = new(proxy.DirectAndProxies(), p => Task.FromResult(p.CreateHttpClient())); }
public ChromeScraper(ProxyCfg proxyCfg, YtCollectCfg collectCfg, ISimpleFileStore logStore) { ProxyCfg = proxyCfg; CollectCfg = collectCfg; LogStore = logStore; ExecutablePath = new(async() => { var revisionInfo = await new BrowserFetcher().DownloadAsync(802497); //revision needs to be recent to be able to use optional chaining return(revisionInfo.ExecutablePath); }); }
public ChromeScraper(ProxyCfg proxyCfg, YtCollectCfg collectCfg, ISimpleFileStore logStore) { ProxyCfg = proxyCfg; CollectCfg = collectCfg; LogStore = logStore; }