private void Login(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; driver.Navigate().GoToUrl(@"https://click.alfabank.ru/"); WaitForPageLoad(chrome.Driver, 2); var name = GetElement(driver, By.TagName("input")); name.Click(); chrome.SendKeys(configuration.Login); chrome.SendKeys(Keys.Return); WaitForPageLoad(chrome.Driver); var pass = GetElement(driver, By.TagName("input")); pass.Click(); chrome.SendKeys(configuration.Password); var smsModel = WaitForSms(() => { chrome.SendKeys(Keys.Return); WaitForPageLoad(driver); }, s => s.Message.ToLower().Contains("пароль для входа")); WaitForPageLoad(chrome.Driver); var code = new string(smsModel.Message.Where(char.IsDigit).ToArray()); chrome.SendKeys(code); chrome.SendKeys(Keys.Return); }
private void Login(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; driver.Navigate().GoToUrl(@"https://www.tinkoff.ru/login/"); WaitForPageLoad(driver); var name = GetElement(driver, By.Name("login")); name.Click(); chrome.SendKeys(configuration.Login); var smsModel = WaitForSms(() => { chrome.SendKeys(Keys.Enter); WaitForPageLoad(driver); }, s => s.Message.Contains("код") && s.Message.Contains("Tinkoff.ru")); var smsCode = GetElement(driver, By.Name("code")); smsCode.Click(); var code = new string(smsModel.Message.Where(char.IsDigit).Take(4).ToArray()); chrome.SendKeys(code); var password = GetElement(driver, By.Name("password")); password.Click(); chrome.SendKeys(configuration.Password); chrome.SendKeys(Keys.Return); WaitForPageLoad(chrome.Driver); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome) { DoLogin(configuration, chrome); var driver = chrome.Driver; var accounts = GetElements(driver, By.ClassName("bank_account")); var result = new List <MoneyStateModel>(); foreach (var acc in accounts) { var title = acc.FindElement(By.ClassName("bank_account_name")).Text; var amount = acc.FindElement(By.ClassName("bank_account_money")).Text; amount = new string(amount.Where(v => char.IsDigit(v) || v == ',').ToArray()); var doubleAmount = double.Parse(amount, new NumberFormatInfo() { NumberDecimalSeparator = "," }); result.Add(Money(title, doubleAmount, CurrencyExtensions.RUB)); } return(result); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; var result = new List <MoneyStateModel>(); driver.Navigate().GoToUrl("https://ru.investing.com/indices/us-spx-500"); var sps = GetElement(driver, By.Id("last_last")).Text; result.Add(Money("SP500", double.Parse(sps, new NumberFormatInfo() { NumberDecimalSeparator = ",", NumberGroupSeparator = "." }), CurrencyExtensions.USD)); foreach (var item in CurrencyExtensions.KnownCurrencies.Where(v => v != CurrencyExtensions.RUB)) { driver.Navigate().GoToUrl($"https://ru.investing.com/currencies/{item.ToLower()}-rub"); sps = GetElement(driver, By.Id("last_last")).Text; var itemRub = item + "/" + CurrencyExtensions.RUB; result.Add(Money(itemRub, double.Parse(sps, new NumberFormatInfo { NumberDecimalSeparator = ",", NumberGroupSeparator = "." }), itemRub)); } return(result); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, ChromeDriver driver) { driver.Navigate().GoToUrl(@"https://online.raiffeisen.ru/"); var name = GetElement(driver, By.Name("username")); var pass = GetElement(driver, By.Name("password")); name.Click(); driver.Keyboard.SendKeys(configuration.Login); pass.Click(); driver.Keyboard.SendKeys(configuration.Password); driver.Keyboard.PressKey(Keys.Return); var amountWait = GetElement(driver, By.ClassName("rc-currency")); var amount = amountWait.Text; var amountClear = new string(amount.Where(v => char.IsDigit(v) || v == ',').ToArray()); var amountNumber = double.Parse(amountClear, new NumberFormatInfo() { NumberDecimalSeparator = "," }); return(new[] { Money("ЗП", amountNumber, "RUB") }); }
private void Login(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; driver.Navigate().GoToUrl(@"https://online.raiffeisen.ru/"); WaitForPageLoad(driver); var name = GetElement(driver, By.ClassName("login-form__username-wrap")).FindElement(By.TagName("input")); var pass = GetElement(driver, By.ClassName("login-form__password-wrap")).FindElement(By.TagName("input")); name.Click(); chrome.SendKeys(configuration.Login); pass.Click(); chrome.SendKeys(configuration.Password); var smsModel = WaitForSms(() => { chrome.SendKeys(Keys.Return); WaitForPageLoad(driver); }, s => s.Message.ToLower().Contains("r-online")); WaitForPageLoad(chrome.Driver); var code = new string(smsModel.Message.Where(char.IsDigit).Take(4).ToArray()); chrome.SendKeys(code); chrome.SendKeys(Keys.Return); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, ChromeDriver driver) { var result = new List <MoneyStateModel>(); driver.Navigate().GoToUrl("https://ru.investing.com/indices/us-spx-500"); var sps = GetElement(driver, By.Id("last_last")).Text; result.Add(Money("SP500", double.Parse(sps, new NumberFormatInfo() { NumberDecimalSeparator = ",", NumberGroupSeparator = "." }), "USD")); driver.Navigate().GoToUrl("https://ru.investing.com/currencies/usd-rub"); sps = GetElement(driver, By.Id("last_last")).Text; result.Add(Money("USD/RUB", double.Parse(sps, new NumberFormatInfo() { NumberDecimalSeparator = ",", NumberGroupSeparator = "." }), "USD/RUB")); driver.Navigate().GoToUrl("https://ru.investing.com/currencies/eur-rub"); sps = GetElement(driver, By.Id("last_last")).Text; result.Add(Money("EUR/RUB", double.Parse(sps, new NumberFormatInfo() { NumberDecimalSeparator = ",", NumberGroupSeparator = "." }), "EUR/RUB")); return(result); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome driver) { var s1 = Calculate(s => s.Amount * (1 + s.Percentage / 100) - s.Returned, "Долги с процентами"); var s2 = Calculate(s => s.Amount - s.Returned, "Долги"); return(s1.Concat(s2).ToList()); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; driver.Navigate().GoToUrl(@"https://my.alfacapital.ru/#/"); var inputs = GetElements(driver, By.TagName("input")); var login = inputs.First(v => v.GetAttribute("name") == "alfa-login"); var password = inputs.First(v => v.GetAttribute("name") == "alfa-password"); login.Click(); chrome.SendKeys(configuration.Login); password.Click(); chrome.SendKeys(configuration.Password); chrome.SendKeys(Keys.Return); var smsModel = WaitForSms(() => {}, s => s.Message.Contains("Код для входа:")); GetElement(driver, By.TagName("input")).Click(); var code = new string(smsModel.Message.Where(char.IsDigit).ToArray()); chrome.SendKeys(code); chrome.SendKeys(Keys.Return); WaitForPageLoad(driver, 5); var result = new List <MoneyStateModel>(); foreach (var tr in GetElements(driver, By.TagName("tr"))) { try { var tds = tr.FindElements(By.TagName("td")).ToList(); var title = tds[1].Text; var value = tds[2].Text; if (title.Contains("\n")) { title = title.Remove(title.IndexOf("\n")).Trim(); } if (title.Equals("Итого", StringComparison.CurrentCultureIgnoreCase)) { continue; } var valueAmount = double.Parse(new string(value.Where(v => char.IsDigit(v) || v == ',').ToArray()), new NumberFormatInfo { NumberDecimalSeparator = "," }); result.Add(Money(title, valueAmount, CurrencyExtensions.RUB)); } catch (Exception ex) {} } return(result); }
public void AddScraper(string name, string login, string password) { var scm = new ScraperConfigurationModel(name) { Login = login, Password = password }; ObjectRepository.Add(scm); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, ChromeDriver driver) { driver.Navigate().GoToUrl(@"https://my.penenza.ru/main/sso/Login.aspx"); var name = GetElement(driver, By.Id("MainContent_txtUserName")); var pass = GetElement(driver, By.Id("MainContent_txtUserPassword")); name.Click(); driver.Keyboard.SendKeys(configuration.Login); pass.Click(); driver.Keyboard.SendKeys(configuration.Password); driver.Keyboard.PressKey(Keys.Return); var firstPortlet = GetElement(driver, By.ClassName("investor-dashboard__portlet-wrapper")); var rows = firstPortlet.FindElements(By.TagName("tr")); Thread.Sleep(10000); var result = new List <MoneyStateModel>(); foreach (var row in rows) { try { var td = row.FindElement(By.ClassName("investor-dashboard__table-title-col")); var value = row.FindElement(By.ClassName("investor-dashboard__table-value-col")); try { var subTd = td.FindElement(By.TagName("a")); if (subTd != null) { td = subTd; } } catch { } var acc = td.Text; var valueText = new string(value.Text.Where(v => char.IsDigit(v) || v == ',').ToArray()); var doubleValue = double.Parse(valueText, new NumberFormatInfo() { NumberDecimalSeparator = "," }); var mm = Money(acc, doubleValue, "RUB"); result.Add(mm); } catch { } } return(result); }
private void Login(ScraperConfigurationModel configuration, ChromeDriver driver) { driver.Navigate().GoToUrl(@"https://click.alfabank.ru/"); var name = GetElement(driver, By.Name("username")); var pass = GetElement(driver, By.Name("password")); name.Click(); driver.Keyboard.SendKeys(configuration.Login); pass.Click(); driver.Keyboard.SendKeys(configuration.Password); driver.Keyboard.PressKey(Keys.Return); }
public IActionResult AddScraper(string name, string login, string password) { var scm = new ScraperConfigurationModel(name) { Login = login, Password = password }; _objectRepository.Add(scm); return(RedirectToAction(nameof(Index))); }
public OkResult AddScraper(string name, string login, string password) { var scm = new ScraperConfigurationModel(name) { Login = login, Password = password }; _objectRepository.Add(scm); return(Ok()); }
private void DoLogin(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; driver.Navigate().GoToUrl(@"https://my.modulbank.ru/"); var name = GetElement(driver, By.Name("login")); name.Click(); foreach (var k in configuration.Login) { chrome.SendKeys(k.ToString()); WaitForPageLoad(driver); } chrome.SendKeys(Keys.Return); var sms = WaitForSms(() => { }, s => s.Message.Contains("Код подтверждения")); var code = new string(sms.Message.Where(char.IsDigit).ToArray()); chrome.SendKeys(code); WaitForPageLoad(driver); var pass = GetElement(driver, By.Name("password")); pass.Click(); chrome.SendKeys(configuration.Password); chrome.SendKeys(Keys.Return); WaitForPageLoad(driver, 15); IEnumerable <IWebElement> popups; do { popups = GetElements(driver, By.ClassName("popup_close")); foreach (var p in popups) { try { p.Click(); } catch { // Ignore underneath popup } } } while (popups.Any()); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome driver) { var s1 = Calculate(s => { var debtJsViewModel = new DebtJsViewModel(s); return(debtJsViewModel.Amount * (1 + debtJsViewModel.Percentage / 100) - debtJsViewModel.Returned); }, "Долги с процентами"); var s2 = Calculate(s => { var vm = new DebtJsViewModel(s); return(vm.Amount - vm.Returned); }, "Долги"); return(s1.Concat(s2).ToList()); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; Login(configuration, chrome); var link = GetElement(driver, By.PartialLinkText("Все счета")); link.Click(); var result = new List <MoneyStateModel>(); var wt = new WebDriverWait(driver, TimeSpan.FromSeconds(30)); wt.Until(x => { try { return(driver.FindElement(By.ClassName("interactiveTable")).FindElements(By.TagName("tr")).Count > 3); } catch { return(false); } }); foreach (var row in GetElement(driver, By.ClassName("interactiveTable")).FindElements(By.TagName("tr"))) { var cells = row.FindElements(By.TagName("td")); var label = cells[0]; var amount = cells[2]; var ccy = cells[3]; var labelText = label.Text; var amountText = amount.Text; var ccyText = ccy.Text; result.Add(Money(labelText, double.Parse(amountText.Replace(" ", ""), new NumberFormatInfo() { NumberDecimalSeparator = "." }), ccyText)); } return(result); }
private void Login(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; driver.Navigate().GoToUrl(@"https://click.alfabank.ru/"); var name = GetElement(driver, By.Name("username")); var pass = GetElement(driver, By.Name("password")); name.Click(); chrome.SendKeys(configuration.Login); pass.Click(); chrome.SendKeys(configuration.Password); chrome.SendKeys(Keys.Return); WaitForPageLoad(chrome.Driver); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, ChromeDriver driver) { driver.Navigate().GoToUrl("https://cabinet.moduldengi.ru/#/"); var auth = GetElement(driver, By.ClassName("auth")); var phone = auth.FindElement(By.TagName("input")); phone.Click(); foreach (var ch in configuration.Login) { driver.Keyboard.SendKeys(ch.ToString()); Thread.Sleep(100); } driver.Keyboard.PressKey(Keys.Enter); Thread.Sleep(1000); var inputs = auth.FindElements(By.TagName("input")); var pass = inputs.First(v => v.GetAttribute("id") != phone.GetAttribute("id")); pass.Click(); driver.Keyboard.SendKeys(configuration.Password); driver.Keyboard.PressKey(Keys.Return); var row = GetElement(driver, By.ClassName("balances-row")); var cells = row.FindElements(By.ClassName("balances-item")); var result = new List <MoneyStateModel>(); foreach (var cell in cells) { var label = cell.FindElement(By.ClassName("property")).Text; var value = cell.FindElement(By.ClassName("value")).Text; var cleanValueString = new string(value.Where(v => char.IsDigit(v) || v == '.').ToArray()); var cleanValue = double.Parse(cleanValueString, new NumberFormatInfo { NumberDecimalSeparator = "." }); result.Add(Money(label, cleanValue, "RUB")); } return(result); }
private void ScrapeCurrentStateImpl(GenericScraper scraper, ScraperConfigurationModel scraperConfig) { var currentState = _objectRepository.Set <MoneyStateModel>(); var accountCount = currentState.Where(s => s.Column.Provider == scraper.ProviderName && s.When.Date == DateTime.UtcNow.Date.AddDays(-1)) .Select(s => s.Column.AccountName).Distinct() .ToList(); var todayState = currentState.Where(s => s.Column.Provider == scraper.ProviderName && s.When.Date == DateTime.UtcNow.Date) .Select(s => s.Column.AccountName).Distinct() .ToList(); var toScrape = accountCount.Count == 0 || accountCount.Except(todayState).Any(); if (toScrape) { _logger.LogInformation("No cached items, scraping..."); var items = scraper.Scrape(scraperConfig, _chrome); _logger.LogInformation($"Found {items.Count()} items, indexing..."); foreach (var item in items) { _logger.LogInformation( $" - {item.Column.Provider} / {item.Column.AccountName}: {item.Amount} ({item.Ccy})"); if (!string.IsNullOrWhiteSpace(item.Column.Provider)) { if (todayState.Contains(item.Column.AccountName)) { continue; } _objectRepository.Add(item); } } scraperConfig.LastSuccessfulBalanceScraping = DateTime.Now; _logger.LogInformation("Indexed..."); } else { _logger.LogInformation("For today there are already scraped items, continuing..."); } }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, ChromeDriver driver) { driver.Navigate().GoToUrl(@"https://potok.digital/potok"); var name = GetElement(driver, By.Name("username")); var pass = GetElement(driver, By.Name("password")); name.Click(); driver.Keyboard.SendKeys(configuration.Login); pass.Click(); driver.Keyboard.SendKeys(configuration.Password); driver.Keyboard.PressKey(Keys.Return); var result = new List <MoneyStateModel>(); var accountTab = GetElement(driver, By.Id("account-tab")); var trs = accountTab.FindElements(By.TagName("tr")); var items = trs.Select(v => v.Text).Where(v => v.Any(char.IsDigit)).ToList(); foreach (var item in items) { var splitPlace = item.IndexOfAny("0123456789".ToCharArray()); var key = item.Remove(splitPlace); var value = item.Substring(splitPlace); if (key.Contains("(")) { key = key.Remove(key.IndexOf("(", StringComparison.Ordinal), key.IndexOf(")", StringComparison.Ordinal) - key.IndexOf("(", StringComparison.Ordinal) + 1); } key = key.Trim(); value = new string(value.Where(v => char.IsDigit(v) || v == ',').ToArray()); var doubleValue = double.Parse(value, new NumberFormatInfo() { NumberDecimalSeparator = "," }); result.Add(Money(key, doubleValue, "RUB")); } return(result); }
private void Login(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; driver.Navigate().GoToUrl(@"https://online.raiffeisen.ru/"); var name = GetElement(driver, By.ClassName("login-form__username-wrap")).FindElement(By.TagName("input")); var pass = GetElement(driver, By.ClassName("login-form__password-wrap")).FindElement(By.TagName("input")); name.Click(); chrome.SendKeys(configuration.Login); pass.Click(); chrome.SendKeys(configuration.Password); chrome.SendKeys(Keys.Return); WaitForPageLoad(driver, 5); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; Login(configuration, chrome); driver.Navigate().GoToUrl(@"https://online.raiffeisen.ru/#/accounts"); var accounts = GetElements(driver, By.TagName("account-widget")); var result = new List <MoneyStateModel>(); foreach (var acc in accounts) { try { var titleElement = acc.FindElement(By.ClassName("product-header-title__name-text")); var text = titleElement.GetAttribute("textContent"); var amountWait = acc.FindElement(By.ClassName("product-header-info__value")); var amount = amountWait.GetAttribute("textContent"); var amountClear = new string(amount.Where(v => char.IsDigit(v) || v == ',').ToArray()); var amountNumber = double.Parse(amountClear, new NumberFormatInfo() { NumberDecimalSeparator = "," }); var ccySign = acc.FindElement(By.ClassName("amount__symbol")); var ccyText = ccySign.GetAttribute("textContent"); result.Add(Money(text, amountNumber, ccyText)); } catch (Exception ex) { Logger.LogError("Failed to parse row, continue", ex); } } return(result); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; Login(configuration, chrome); var links = GetElements(driver, By.TagName("a")).ToList(); var matchingLinks = links.Where(v => v.GetProperty("href").Contains("/events/account/")).ToList(); var result = new List <MoneyStateModel>(); foreach (var link in matchingLinks) { var children = link.FindElements(By.TagName("div")); var name = children.FirstOrDefault(v => v.GetAttribute("class").Contains("Item__name")); var balance = children.FirstOrDefault(v => v.GetAttribute("class").Contains("Item__balance")); if (name == null || balance == null) { continue; } var nameText = name.Text; var balanceText = balance.Text; var balanceTextClear = new string(balanceText.Where(v => char.IsDigit(v) || v == ',').ToArray()); var balanceValue = double.Parse(balanceTextClear.Replace(" ", "")); var ccy = "RUB"; if (balanceText.Contains("$")) { ccy = "USD"; } if (balanceText.Contains("€")) { ccy = "EUR"; } result.Add(Money(nameText, balanceValue, ccy)); } return(result); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome driver) { using var connection = ConnectionFactory.GetConnection(configuration.Password); using var context = connection.Context; var accounts = context.AccountsAsync().GetAwaiter().GetResult(); var result = new List <MoneyStateModel>(); foreach (var account in accounts) { var portfolio = context.PortfolioAsync(account.BrokerAccountId).GetAwaiter().GetResult(); result.AddRange(portfolio.Positions.Select(v => Money(account.BrokerAccountId + " " + v.Name, (double)(v.Balance * v.AveragePositionPrice.Value), v.AveragePositionPrice.Currency.ToString().ToUpper()))); var ccys = context.PortfolioCurrenciesAsync(account.BrokerAccountId).GetAwaiter().GetResult(); result.AddRange(ccys.Currencies.Select(v => Money(account.BrokerAccountId + " " + v.Currency.ToString().ToUpper(), (double)v.Balance, v.Currency.ToString().ToUpper()))); } return(result); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; var result = new List <MoneyStateModel>(); driver.Navigate().GoToUrl("https://ru.investing.com/indices/us-spx-500"); var sps = GetElement(driver, By.Id("last_last")).Text; result.Add(Money("SP500", double.Parse(sps, new NumberFormatInfo() { NumberDecimalSeparator = ",", NumberGroupSeparator = "." }), CurrencyExtensions.USD)); driver.Navigate().GoToUrl("https://ru.investing.com/currencies/usd-rub"); sps = GetElement(driver, By.Id("last_last")).Text; var usdRub = CurrencyExtensions.USD + "/" + CurrencyExtensions.RUB; result.Add(Money(usdRub, double.Parse(sps, new NumberFormatInfo() { NumberDecimalSeparator = ",", NumberGroupSeparator = "." }), usdRub)); driver.Navigate().GoToUrl("https://ru.investing.com/currencies/eur-rub"); sps = GetElement(driver, By.Id("last_last")).Text; var eurRub = CurrencyExtensions.EUR + "/" + CurrencyExtensions.RUB; result.Add(Money(eurRub, double.Parse(sps, new NumberFormatInfo() { NumberDecimalSeparator = ",", NumberGroupSeparator = "." }), eurRub)); return(result); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; var result = new List <MoneyStateModel>(); driver.Navigate().GoToUrl("https://eodhistoricaldata.com/financial-summary/GSPC.INDX"); WaitForPageLoad(driver); var moneyStateModel = ParseMoney("SP500", driver); result.Add(moneyStateModel); foreach (var item in CurrencyExtensions.KnownCurrencies.Where(v => v != CurrencyExtensions.RUB)) { var url = $"https://eodhistoricaldata.com/financial-summary/{item.ToUpper()}RUB.FOREX"; driver.Navigate().GoToUrl(url); Logger.LogInformation($"Scraping {url}"); var itemRub = item + "/" + CurrencyExtensions.RUB; var msm = ParseMoney(itemRub, driver); result.Add(msm); } return(result); }
public IEnumerable <PaymentModel> ScrapeStatement(ScraperConfigurationModel configuration, Chrome chromeDriver, DateTime startFrom) { if (startFrom < DateTime.Now.AddYears(-2).AddDays(1)) { startFrom = DateTime.Now.AddYears(-2).AddDays(1); } var driver = chromeDriver.Driver; Login(configuration, driver); var existingLinks = driver.FindElementsByPartialLinkText("История операций").ToList(); var link1 = (RemoteWebElement)driver.FindElementByLinkText("Счета"); driver.Mouse.MouseMove(link1.Coordinates, 1, 1); Thread.Sleep(500); existingLinks = driver.FindElementsByPartialLinkText("История операций").Except(existingLinks).ToList(); existingLinks.Single().Click(); Thread.Sleep(2000); var selectBtn = driver.FindElementById("pt1:soc1::button"); var accountsChooser = driver.FindElementById("pt1:soc1::pop"); var accs = accountsChooser.FindElements(By.TagName("tr")); var result = new List <PaymentModel>(); foreach (var acc in accs) { var tds = acc.FindElements(By.TagName("td")); if (tds.Count < 4) { continue; } selectBtn.Click(); Thread.Sleep(2000); tds[0].Click(); Thread.Sleep(2000); var inputDate = driver.FindElementById("pt1:id1::fd"); inputDate.Click(); Thread.Sleep(2000); driver.Keyboard.SendKeys(Enumerable.Repeat(Keys.Delete, 20).Join("")); Thread.Sleep(500); driver.Keyboard.SendKeys(Enumerable.Repeat(Keys.Backspace, 20).Join("")); Thread.Sleep(500); driver.Keyboard.SendKeys(Enumerable.Repeat(Keys.Delete, 20).Join("")); Thread.Sleep(500); driver.Keyboard.SendKeys(Enumerable.Repeat(Keys.Backspace, 20).Join("")); Thread.Sleep(500); driver.Keyboard.SendKeys(startFrom.ToString("ddMMyyyy")); var submit = driver.FindElementById("pt1:showButton::button"); submit.Click(); Thread.Sleep(2000); var csv = driver.FindElementById("pt1:downloadCSVLink"); csv.Click(); int waited = 0; while (chromeDriver.GetDownloads().Count < 1 && waited < 300) { Thread.Sleep(1000); waited++; } var files = chromeDriver.GetDownloads(); if (files.Count == 1) { var csvFile = files.First(); var csvContent = File.ReadAllLines(csvFile.FullName, Encoding.GetEncoding(1251)).Skip(1).Select(v => new AlphaStatement(v)).ToList(); var payments = csvContent.Select(v => Statement(v.Date, v.AccountName, v.What, v.Outcome - v.Income, v.Ccy, v.Reference)).ToList(); var holdPayments = payments.Where(v => v.StatementReference == "HOLD").ToList(); payments = payments.Except(holdPayments).ToList(); result.AddRange(payments); csvFile.Delete(); } chromeDriver.CleanupDownloads(); } return(result); }
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome) { var driver = chrome.Driver; driver.Navigate().GoToUrl("https://cabinet.moduldengi.ru/#/"); var auth = GetElement(driver, By.ClassName("auth")); var phone = auth.FindElement(By.TagName("input")); phone.Click(); foreach (var ch in configuration.Login) { chrome.SendKeys(ch.ToString()); WaitForPageLoad(driver); } chrome.SendKeys(Keys.Enter); WaitForPageLoad(driver, 5); var inputs = auth.FindElements(By.TagName("input")); var pass = inputs.First(v => v.GetAttribute("id") != phone.GetAttribute("id")); pass.Click(); chrome.SendKeys(configuration.Password); chrome.SendKeys(Keys.Return); WaitForPageLoad(driver, 5); var row = GetElement(driver, By.ClassName("balances-row")); var cells = row.FindElements(By.ClassName("balances-item")); var result = new List <MoneyStateModel>(); foreach (var cell in cells) { var label = cell.FindElement(By.ClassName("property")).Text; var value = cell.FindElement(By.ClassName("value")).Text; var cleanValue = ParseDouble(value); result.Add(Money(label, cleanValue, CurrencyExtensions.RUB)); } var investmentsSection = GetElement(driver, By.ClassName("investments")); while (investmentsSection.Text.ToLower().Contains("загрузка")) { WaitForPageLoad(driver); } var blockMenu = investmentsSection.FindElement(By.ClassName("block-menu")); var links = blockMenu.FindElements(By.TagName("a")); var badOnesButton = links.First(v => v.Text.ToLower() == "просроченные"); badOnesButton.Click(); WaitForPageLoad(driver); var badProjects = investmentsSection.FindElements(By.ClassName("project-item-wrapper")); var titles = badProjects.Select(v => v.FindElement(By.ClassName("project-table"))).ToList(); var spans = titles.Select(v => { var rows = v.FindElements(By.TagName("tr")).Select(s => s.FindElements(By.TagName("td")).ToList()).ToList(); var parsed = rows.Select(s => { var key = s[0].Text; var value = s[1]; var spanValue = ParseDouble(value.FindElement(By.TagName("span")).Text); return(new { key, spanValue }); }).ToList(); var inv = parsed.First(s => s.key.ToLower().Contains("инвестиция")).spanValue; var ok = parsed.First(s => s.key.ToLower().Contains("погашено")).spanValue; return(inv - ok); }).ToList(); result.Add(Money("Просрочки", spans.Sum(), CurrencyExtensions.RUB)); return(result); }
public override IList <PaymentModel> ScrapeStatement(ScraperConfigurationModel configuration, Chrome chrome, DateTime startFrom) { var driver = chrome.Driver; Login(configuration, chrome); driver.Navigate().GoToUrl(@"https://online.raiffeisen.ru/#/history/statement"); var accounts = GetElements(driver, By.TagName("c-select-option-account")); var link = GetElements(driver, By.TagName("a")).First(v => v.GetAttribute("href")?.Contains("/transaction.ofx?") == true); var linkText = link.GetAttribute("href"); var build = new Uri(linkText); var result = new List <PaymentModel>(); var urlFormat = @"https://online.raiffeisen.ru/rest/account/{accountId}/transaction.ofx?from={from}&to={to}&sort=date&order=desc&access_token={token}"; var originalQuery = QueryHelpers.ParseQuery(build.Query); var accessToken = originalQuery["access_token"].First(); var accountDetails = accounts.Select(v => { var id = v.FindElement(By.TagName("div")).GetAttribute("data-account-id"); var textElement = v.FindElement(By.TagName("account-logo")).FindElement(By.XPath("..")); var name = textElement.GetAttribute("textContent").Trim(); return(id, name); }).Distinct().ToList(); Logger.LogInformation($"Found {accountDetails.Count} Raiffeisen accounts"); foreach (var account in accountDetails) { var accountId = account.id; var accountName = account.name; var url = urlFormat.Replace("{accountId}", accountId) .Replace("{from}", startFrom.ToString("yyyy-MM-ddTHH:mm")) .Replace("{to}", DateTime.Now.ToString("yyyy-MM-ddTHH:mm")) .Replace("{token}", accessToken); // Raiffeisen sometimes doesn't return all payments on first call for (int i = 0; i < 3; i++) { driver.Navigate().GoToUrl(url); Logger.LogInformation($"Getting statement for {account.name} at {url}, attempt {i}"); int waited = 0; while (chrome.GetDownloads().Count < 1 && waited < 300) { WaitForPageLoad(driver); waited++; } Thread.Sleep(10000); var files = chrome.GetDownloads(); if (files.Count == 1) { var ofxFile = files.First(); var doc = File.ReadAllText(ofxFile.FullName); var xdoc = XDocument.Parse(doc); var statements = xdoc.XPathSelectElements("OFX/BANKMSGSRSV1/STMTTRNRS/STMTRS/BANKTRANLIST/STMTTRN"); var ccyNode = xdoc.XPathSelectElement("OFX/BANKMSGSRSV1/STMTTRNRS/STMTRS/CURDEF"); var ccy = ccyNode.Value; var payments = new List <PaymentModel>(); foreach (var st in statements) { var timeStr = st.Element("DTPOSTED").Value; var time = DateTime.ParseExact(timeStr, "yyyyMMddhhmmss", CultureInfo.CurrentCulture); var amountStr = st.Element("TRNAMT").Value; var amount = double.Parse(amountStr, new NumberFormatInfo { NumberDecimalSeparator = "." }); var name = st.Element("MEMO").Value; var id = timeStr + amountStr + name; int counter = 0; while (payments.Any(j => j.StatementReference == id)) { id = timeStr + amountStr + name + counter++; } var kind = amount < 0 ? PaymentKind.Expense : PaymentKind.Income; var stmt = Statement(time, accountName, name, amount, kind, ccy, id); payments.Add(stmt); } Logger.LogInformation($"Got {payments.Count} payments from {url}, attempt {i}"); result.AddRange(payments); ofxFile.Delete(); } } } var oldCount = result.Count; result = result.GroupBy(v => v.StatementReference).Select(v => v.First()).ToList(); Logger.LogInformation($"Deduplicated payments {oldCount} -> {result.Count}"); return(result); }