public WebScraper(ProxyCfg proxy, YtCollectCfg collectCfg, ISimpleFileStore logStore) { Proxy = proxy; CollectCfg = collectCfg; LogStore = logStore; Clients = new ResourceCycle <HttpClient, ProxyConnectionCfg>(proxy.DirectAndProxies(), p => Task.FromResult(CreateHttpClient(p))); }
public YtWeb(ProxyCfg proxy, YtCollectCfg collectCfg, ISimpleFileStore logStore) { Proxy = proxy; CollectCfg = collectCfg; LogStore = logStore; Clients = new(proxy.DirectAndProxies(), p => Task.FromResult(p.CreateHttpClient())); }
public ChromeScraper(ProxyCfg proxyCfg, YtCollectCfg collectCfg, ISimpleFileStore logStore) { ProxyCfg = proxyCfg; CollectCfg = collectCfg; LogStore = logStore; ExecutablePath = new(async() => { var revisionInfo = await new BrowserFetcher().DownloadAsync(802497); //revision needs to be recent to be able to use optional chaining return(revisionInfo.ExecutablePath); }); }
public static ProxyConnectionCfg Proxy(this ProxyCfg cfg, ProxyType type) => cfg.Proxies.FirstOrDefault(c => c.Type == type);
public BcWeb(ProxyCfg proxy, BitChuteCfg cfg) { Proxy = proxy; Cfg = cfg; FlurlClients = new(new(), new(proxy.Proxies.FirstOrDefault()?.CreateHttpClient())); }
public ChromeScraper(ProxyCfg proxyCfg, YtCollectCfg collectCfg, ISimpleFileStore logStore) { ProxyCfg = proxyCfg; CollectCfg = collectCfg; LogStore = logStore; }