public Feed TryScrapeFeed(ScraperConfig config) { CookieContainer cookieJar = new CookieContainer(); var cookies = WebDriver.Manage().Cookies.AllCookies; foreach (var cookie in cookies) { cookieJar.Add(new Cookie(cookie.Name, cookie.Value, cookie.Path, cookie.Domain)); } int maxRetries = 5; int retry = 0; Logger.Trace("Downloading {0}", config.ApiUrl); Feed feed = null; while (true) { if (retry >= maxRetries) { throw new ScraperException("Can't download project feed. Retries exceeded", config); } try { using (var client = new CookieWebClient(cookieJar)) { if (!ProxyAddress.IsNullOrEmpty()) { client.Proxy = new WebProxy(ProxyAddress); } var mozilaAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36"; client.Headers.Add("User-Agent", mozilaAgent); // Call authenticated resources string result = client.DownloadString(config.ApiUrl); feed = JsonConvert.DeserializeObject <Feed>(result); } break; } catch (Exception e) { Logger.Trace("Failed to download {0} page. Retry {1} of {2}", config.ApiUrl, retry, maxRetries); Logger.Error(e); retry++; Task.Delay(TimeSpan.FromSeconds(10)).Wait(); } } Logger.Info("Successfully downloaded {0}", config.ApiUrl); return(feed); }
public void Login(ScraperConfig config) { var userNameInput = FindElementWaitSafe(By.CssSelector("#username"), 1); var passwordInput = FindElementWaitSafe(By.CssSelector("#passwd"), 1); var loginBtn = FindElementWaitSafe(By.CssSelector("#login_btn"), 1); if (userNameInput == null || passwordInput == null || loginBtn == null) { throw new ScraperException("Invalid login page", config); } userNameInput.Clear(); passwordInput.Clear(); userNameInput.SendKeys(config.UserName); passwordInput.SendKeys(config.Password); loginBtn.Click(); }
public void GoToDashboard(ScraperConfig config) { int maxRetries = 5; int retry = 0; while (retry < maxRetries) { GoToUrl(config.Url); if (Regex.IsMatch(WebDriver.Url, @"/login") || WebDriver.Url == config.Url) { break; } retry++; } if (WebDriver.Url != config.Url) { Login(config); } }
public ScraperException(string message, ScraperConfig config) : base(message) { Config = config; }