private void Login(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
driver.Navigate().GoToUrl(@"https://click.alfabank.ru/");
WaitForPageLoad(chrome.Driver, 2);
var name = GetElement(driver, By.TagName("input"));
name.Click();
chrome.SendKeys(configuration.Login);
chrome.SendKeys(Keys.Return);
WaitForPageLoad(chrome.Driver);
var pass = GetElement(driver, By.TagName("input"));
pass.Click();
chrome.SendKeys(configuration.Password);
var smsModel = WaitForSms(() =>
{
chrome.SendKeys(Keys.Return);
WaitForPageLoad(driver);
}, s => s.Message.ToLower().Contains("пароль для входа"));
WaitForPageLoad(chrome.Driver);
var code = new string(smsModel.Message.Where(char.IsDigit).ToArray());
chrome.SendKeys(code);
chrome.SendKeys(Keys.Return);
}
private void Login(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
driver.Navigate().GoToUrl(@"https://www.tinkoff.ru/login/");
WaitForPageLoad(driver);
var name = GetElement(driver, By.Name("login"));
name.Click();
chrome.SendKeys(configuration.Login);
var smsModel = WaitForSms(() =>
{
chrome.SendKeys(Keys.Enter);
WaitForPageLoad(driver);
}, s => s.Message.Contains("код") && s.Message.Contains("Tinkoff.ru"));
var smsCode = GetElement(driver, By.Name("code"));
smsCode.Click();
var code = new string(smsModel.Message.Where(char.IsDigit).Take(4).ToArray());
chrome.SendKeys(code);
var password = GetElement(driver, By.Name("password"));
password.Click();
chrome.SendKeys(configuration.Password);
chrome.SendKeys(Keys.Return);
WaitForPageLoad(chrome.Driver);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome)
{
DoLogin(configuration, chrome);
var driver = chrome.Driver;
var accounts = GetElements(driver, By.ClassName("bank_account"));
var result = new List <MoneyStateModel>();
foreach (var acc in accounts)
{
var title = acc.FindElement(By.ClassName("bank_account_name")).Text;
var amount = acc.FindElement(By.ClassName("bank_account_money")).Text;
amount = new string(amount.Where(v => char.IsDigit(v) || v == ',').ToArray());
var doubleAmount = double.Parse(amount, new NumberFormatInfo()
{
NumberDecimalSeparator = ","
});
result.Add(Money(title, doubleAmount, CurrencyExtensions.RUB));
}
return(result);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
var result = new List <MoneyStateModel>();
driver.Navigate().GoToUrl("https://ru.investing.com/indices/us-spx-500");
var sps = GetElement(driver, By.Id("last_last")).Text;
result.Add(Money("SP500",
double.Parse(sps, new NumberFormatInfo()
{
NumberDecimalSeparator = ",", NumberGroupSeparator = "."
}),
CurrencyExtensions.USD));
foreach (var item in CurrencyExtensions.KnownCurrencies.Where(v => v != CurrencyExtensions.RUB))
{
driver.Navigate().GoToUrl($"https://ru.investing.com/currencies/{item.ToLower()}-rub");
sps = GetElement(driver, By.Id("last_last")).Text;
var itemRub = item + "/" + CurrencyExtensions.RUB;
result.Add(Money(itemRub,
double.Parse(sps, new NumberFormatInfo {
NumberDecimalSeparator = ",", NumberGroupSeparator = "."
}),
itemRub));
}
return(result);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, ChromeDriver driver)
{
driver.Navigate().GoToUrl(@"https://online.raiffeisen.ru/");
var name = GetElement(driver, By.Name("username"));
var pass = GetElement(driver, By.Name("password"));
name.Click();
driver.Keyboard.SendKeys(configuration.Login);
pass.Click();
driver.Keyboard.SendKeys(configuration.Password);
driver.Keyboard.PressKey(Keys.Return);
var amountWait = GetElement(driver, By.ClassName("rc-currency"));
var amount = amountWait.Text;
var amountClear = new string(amount.Where(v => char.IsDigit(v) || v == ',').ToArray());
var amountNumber = double.Parse(amountClear, new NumberFormatInfo()
{
NumberDecimalSeparator = ","
});
return(new[]
{
Money("ЗП", amountNumber, "RUB")
});
}
private void Login(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
driver.Navigate().GoToUrl(@"https://online.raiffeisen.ru/");
WaitForPageLoad(driver);
var name = GetElement(driver, By.ClassName("login-form__username-wrap")).FindElement(By.TagName("input"));
var pass = GetElement(driver, By.ClassName("login-form__password-wrap")).FindElement(By.TagName("input"));
name.Click();
chrome.SendKeys(configuration.Login);
pass.Click();
chrome.SendKeys(configuration.Password);
var smsModel = WaitForSms(() =>
{
chrome.SendKeys(Keys.Return);
WaitForPageLoad(driver);
}, s => s.Message.ToLower().Contains("r-online"));
WaitForPageLoad(chrome.Driver);
var code = new string(smsModel.Message.Where(char.IsDigit).Take(4).ToArray());
chrome.SendKeys(code);
chrome.SendKeys(Keys.Return);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, ChromeDriver driver)
{
var result = new List <MoneyStateModel>();
driver.Navigate().GoToUrl("https://ru.investing.com/indices/us-spx-500");
var sps = GetElement(driver, By.Id("last_last")).Text;
result.Add(Money("SP500",
double.Parse(sps, new NumberFormatInfo()
{
NumberDecimalSeparator = ",", NumberGroupSeparator = "."
}),
"USD"));
driver.Navigate().GoToUrl("https://ru.investing.com/currencies/usd-rub");
sps = GetElement(driver, By.Id("last_last")).Text;
result.Add(Money("USD/RUB",
double.Parse(sps, new NumberFormatInfo()
{
NumberDecimalSeparator = ",", NumberGroupSeparator = "."
}),
"USD/RUB"));
driver.Navigate().GoToUrl("https://ru.investing.com/currencies/eur-rub");
sps = GetElement(driver, By.Id("last_last")).Text;
result.Add(Money("EUR/RUB", double.Parse(sps,
new NumberFormatInfo()
{
NumberDecimalSeparator = ",", NumberGroupSeparator = "."
}), "EUR/RUB"));
return(result);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome driver)
{
var s1 = Calculate(s => s.Amount * (1 + s.Percentage / 100) - s.Returned, "Долги с процентами");
var s2 = Calculate(s => s.Amount - s.Returned, "Долги");
return(s1.Concat(s2).ToList());
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
driver.Navigate().GoToUrl(@"https://my.alfacapital.ru/#/");
var inputs = GetElements(driver, By.TagName("input"));
var login = inputs.First(v => v.GetAttribute("name") == "alfa-login");
var password = inputs.First(v => v.GetAttribute("name") == "alfa-password");
login.Click();
chrome.SendKeys(configuration.Login);
password.Click();
chrome.SendKeys(configuration.Password);
chrome.SendKeys(Keys.Return);
var smsModel = WaitForSms(() => {}, s => s.Message.Contains("Код для входа:"));
GetElement(driver, By.TagName("input")).Click();
var code = new string(smsModel.Message.Where(char.IsDigit).ToArray());
chrome.SendKeys(code);
chrome.SendKeys(Keys.Return);
WaitForPageLoad(driver, 5);
var result = new List <MoneyStateModel>();
foreach (var tr in GetElements(driver, By.TagName("tr")))
{
try
{
var tds = tr.FindElements(By.TagName("td")).ToList();
var title = tds[1].Text;
var value = tds[2].Text;
if (title.Contains("\n"))
{
title = title.Remove(title.IndexOf("\n")).Trim();
}
if (title.Equals("Итого", StringComparison.CurrentCultureIgnoreCase))
{
continue;
}
var valueAmount = double.Parse(new string(value.Where(v => char.IsDigit(v) || v == ',').ToArray()),
new NumberFormatInfo {
NumberDecimalSeparator = ","
});
result.Add(Money(title, valueAmount, CurrencyExtensions.RUB));
}
catch (Exception ex)
{}
}
return(result);
}
public void AddScraper(string name, string login, string password)
{
var scm = new ScraperConfigurationModel(name)
{
Login = login,
Password = password
};
ObjectRepository.Add(scm);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, ChromeDriver driver)
{
driver.Navigate().GoToUrl(@"https://my.penenza.ru/main/sso/Login.aspx");
var name = GetElement(driver, By.Id("MainContent_txtUserName"));
var pass = GetElement(driver, By.Id("MainContent_txtUserPassword"));
name.Click();
driver.Keyboard.SendKeys(configuration.Login);
pass.Click();
driver.Keyboard.SendKeys(configuration.Password);
driver.Keyboard.PressKey(Keys.Return);
var firstPortlet = GetElement(driver, By.ClassName("investor-dashboard__portlet-wrapper"));
var rows = firstPortlet.FindElements(By.TagName("tr"));
Thread.Sleep(10000);
var result = new List <MoneyStateModel>();
foreach (var row in rows)
{
try
{
var td = row.FindElement(By.ClassName("investor-dashboard__table-title-col"));
var value = row.FindElement(By.ClassName("investor-dashboard__table-value-col"));
try
{
var subTd = td.FindElement(By.TagName("a"));
if (subTd != null)
{
td = subTd;
}
}
catch
{
}
var acc = td.Text;
var valueText = new string(value.Text.Where(v => char.IsDigit(v) || v == ',').ToArray());
var doubleValue = double.Parse(valueText, new NumberFormatInfo()
{
NumberDecimalSeparator = ","
});
var mm = Money(acc, doubleValue, "RUB");
result.Add(mm);
}
catch
{
}
}
return(result);
}
private void Login(ScraperConfigurationModel configuration, ChromeDriver driver)
{
driver.Navigate().GoToUrl(@"https://click.alfabank.ru/");
var name = GetElement(driver, By.Name("username"));
var pass = GetElement(driver, By.Name("password"));
name.Click();
driver.Keyboard.SendKeys(configuration.Login);
pass.Click();
driver.Keyboard.SendKeys(configuration.Password);
driver.Keyboard.PressKey(Keys.Return);
}
public IActionResult AddScraper(string name, string login, string password)
{
var scm = new ScraperConfigurationModel(name)
{
Login = login,
Password = password
};
_objectRepository.Add(scm);
return(RedirectToAction(nameof(Index)));
}
public OkResult AddScraper(string name, string login, string password)
{
var scm = new ScraperConfigurationModel(name)
{
Login = login,
Password = password
};
_objectRepository.Add(scm);
return(Ok());
}
private void DoLogin(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
driver.Navigate().GoToUrl(@"https://my.modulbank.ru/");
var name = GetElement(driver, By.Name("login"));
name.Click();
foreach (var k in configuration.Login)
{
chrome.SendKeys(k.ToString());
WaitForPageLoad(driver);
}
chrome.SendKeys(Keys.Return);
var sms = WaitForSms(() => { },
s => s.Message.Contains("Код подтверждения"));
var code = new string(sms.Message.Where(char.IsDigit).ToArray());
chrome.SendKeys(code);
WaitForPageLoad(driver);
var pass = GetElement(driver, By.Name("password"));
pass.Click();
chrome.SendKeys(configuration.Password);
chrome.SendKeys(Keys.Return);
WaitForPageLoad(driver, 15);
IEnumerable <IWebElement> popups;
do
{
popups = GetElements(driver, By.ClassName("popup_close"));
foreach (var p in popups)
{
try
{
p.Click();
}
catch
{
// Ignore underneath popup
}
}
} while (popups.Any());
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome driver)
{
var s1 = Calculate(s =>
{
var debtJsViewModel = new DebtJsViewModel(s);
return(debtJsViewModel.Amount * (1 + debtJsViewModel.Percentage / 100) - debtJsViewModel.Returned);
}, "Долги с процентами");
var s2 = Calculate(s =>
{
var vm = new DebtJsViewModel(s);
return(vm.Amount - vm.Returned);
}, "Долги");
return(s1.Concat(s2).ToList());
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
Login(configuration, chrome);
var link = GetElement(driver, By.PartialLinkText("Все счета"));
link.Click();
var result = new List <MoneyStateModel>();
var wt = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
wt.Until(x =>
{
try
{
return(driver.FindElement(By.ClassName("interactiveTable")).FindElements(By.TagName("tr")).Count > 3);
}
catch
{
return(false);
}
});
foreach (var row in GetElement(driver, By.ClassName("interactiveTable")).FindElements(By.TagName("tr")))
{
var cells = row.FindElements(By.TagName("td"));
var label = cells[0];
var amount = cells[2];
var ccy = cells[3];
var labelText = label.Text;
var amountText = amount.Text;
var ccyText = ccy.Text;
result.Add(Money(labelText,
double.Parse(amountText.Replace(" ", ""), new NumberFormatInfo()
{
NumberDecimalSeparator = "."
}),
ccyText));
}
return(result);
}
private void Login(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
driver.Navigate().GoToUrl(@"https://click.alfabank.ru/");
var name = GetElement(driver, By.Name("username"));
var pass = GetElement(driver, By.Name("password"));
name.Click();
chrome.SendKeys(configuration.Login);
pass.Click();
chrome.SendKeys(configuration.Password);
chrome.SendKeys(Keys.Return);
WaitForPageLoad(chrome.Driver);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, ChromeDriver driver)
{
driver.Navigate().GoToUrl("https://cabinet.moduldengi.ru/#/");
var auth = GetElement(driver, By.ClassName("auth"));
var phone = auth.FindElement(By.TagName("input"));
phone.Click();
foreach (var ch in configuration.Login)
{
driver.Keyboard.SendKeys(ch.ToString());
Thread.Sleep(100);
}
driver.Keyboard.PressKey(Keys.Enter);
Thread.Sleep(1000);
var inputs = auth.FindElements(By.TagName("input"));
var pass = inputs.First(v => v.GetAttribute("id") != phone.GetAttribute("id"));
pass.Click();
driver.Keyboard.SendKeys(configuration.Password);
driver.Keyboard.PressKey(Keys.Return);
var row = GetElement(driver, By.ClassName("balances-row"));
var cells = row.FindElements(By.ClassName("balances-item"));
var result = new List <MoneyStateModel>();
foreach (var cell in cells)
{
var label = cell.FindElement(By.ClassName("property")).Text;
var value = cell.FindElement(By.ClassName("value")).Text;
var cleanValueString = new string(value.Where(v => char.IsDigit(v) || v == '.').ToArray());
var cleanValue = double.Parse(cleanValueString, new NumberFormatInfo
{
NumberDecimalSeparator = "."
});
result.Add(Money(label, cleanValue, "RUB"));
}
return(result);
}
private void ScrapeCurrentStateImpl(GenericScraper scraper, ScraperConfigurationModel scraperConfig)
{
var currentState = _objectRepository.Set <MoneyStateModel>();
var accountCount = currentState.Where(s =>
s.Column.Provider == scraper.ProviderName && s.When.Date == DateTime.UtcNow.Date.AddDays(-1))
.Select(s => s.Column.AccountName).Distinct()
.ToList();
var todayState = currentState.Where(s =>
s.Column.Provider == scraper.ProviderName && s.When.Date == DateTime.UtcNow.Date)
.Select(s => s.Column.AccountName).Distinct()
.ToList();
var toScrape = accountCount.Count == 0 || accountCount.Except(todayState).Any();
if (toScrape)
{
_logger.LogInformation("No cached items, scraping...");
var items = scraper.Scrape(scraperConfig, _chrome);
_logger.LogInformation($"Found {items.Count()} items, indexing...");
foreach (var item in items)
{
_logger.LogInformation(
$" - {item.Column.Provider} / {item.Column.AccountName}: {item.Amount} ({item.Ccy})");
if (!string.IsNullOrWhiteSpace(item.Column.Provider))
{
if (todayState.Contains(item.Column.AccountName))
{
continue;
}
_objectRepository.Add(item);
}
}
scraperConfig.LastSuccessfulBalanceScraping = DateTime.Now;
_logger.LogInformation("Indexed...");
}
else
{
_logger.LogInformation("For today there are already scraped items, continuing...");
}
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, ChromeDriver driver)
{
driver.Navigate().GoToUrl(@"https://potok.digital/potok");
var name = GetElement(driver, By.Name("username"));
var pass = GetElement(driver, By.Name("password"));
name.Click();
driver.Keyboard.SendKeys(configuration.Login);
pass.Click();
driver.Keyboard.SendKeys(configuration.Password);
driver.Keyboard.PressKey(Keys.Return);
var result = new List <MoneyStateModel>();
var accountTab = GetElement(driver, By.Id("account-tab"));
var trs = accountTab.FindElements(By.TagName("tr"));
var items = trs.Select(v => v.Text).Where(v => v.Any(char.IsDigit)).ToList();
foreach (var item in items)
{
var splitPlace = item.IndexOfAny("0123456789".ToCharArray());
var key = item.Remove(splitPlace);
var value = item.Substring(splitPlace);
if (key.Contains("("))
{
key = key.Remove(key.IndexOf("(", StringComparison.Ordinal), key.IndexOf(")", StringComparison.Ordinal) - key.IndexOf("(", StringComparison.Ordinal) + 1);
}
key = key.Trim();
value = new string(value.Where(v => char.IsDigit(v) || v == ',').ToArray());
var doubleValue = double.Parse(value, new NumberFormatInfo()
{
NumberDecimalSeparator = ","
});
result.Add(Money(key, doubleValue, "RUB"));
}
return(result);
}
private void Login(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
driver.Navigate().GoToUrl(@"https://online.raiffeisen.ru/");
var name = GetElement(driver, By.ClassName("login-form__username-wrap")).FindElement(By.TagName("input"));
var pass = GetElement(driver, By.ClassName("login-form__password-wrap")).FindElement(By.TagName("input"));
name.Click();
chrome.SendKeys(configuration.Login);
pass.Click();
chrome.SendKeys(configuration.Password);
chrome.SendKeys(Keys.Return);
WaitForPageLoad(driver, 5);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
Login(configuration, chrome);
driver.Navigate().GoToUrl(@"https://online.raiffeisen.ru/#/accounts");
var accounts = GetElements(driver, By.TagName("account-widget"));
var result = new List <MoneyStateModel>();
foreach (var acc in accounts)
{
try
{
var titleElement = acc.FindElement(By.ClassName("product-header-title__name-text"));
var text = titleElement.GetAttribute("textContent");
var amountWait = acc.FindElement(By.ClassName("product-header-info__value"));
var amount = amountWait.GetAttribute("textContent");
var amountClear = new string(amount.Where(v => char.IsDigit(v) || v == ',').ToArray());
var amountNumber = double.Parse(amountClear, new NumberFormatInfo()
{
NumberDecimalSeparator = ","
});
var ccySign = acc.FindElement(By.ClassName("amount__symbol"));
var ccyText = ccySign.GetAttribute("textContent");
result.Add(Money(text, amountNumber, ccyText));
}
catch (Exception ex)
{
Logger.LogError("Failed to parse row, continue", ex);
}
}
return(result);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
Login(configuration, chrome);
var links = GetElements(driver, By.TagName("a")).ToList();
var matchingLinks = links.Where(v => v.GetProperty("href").Contains("/events/account/")).ToList();
var result = new List <MoneyStateModel>();
foreach (var link in matchingLinks)
{
var children = link.FindElements(By.TagName("div"));
var name = children.FirstOrDefault(v => v.GetAttribute("class").Contains("Item__name"));
var balance = children.FirstOrDefault(v => v.GetAttribute("class").Contains("Item__balance"));
if (name == null || balance == null)
{
continue;
}
var nameText = name.Text;
var balanceText = balance.Text;
var balanceTextClear = new string(balanceText.Where(v => char.IsDigit(v) || v == ',').ToArray());
var balanceValue = double.Parse(balanceTextClear.Replace(" ", ""));
var ccy = "RUB";
if (balanceText.Contains("$"))
{
ccy = "USD";
}
if (balanceText.Contains("€"))
{
ccy = "EUR";
}
result.Add(Money(nameText, balanceValue, ccy));
}
return(result);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome driver)
{
using var connection = ConnectionFactory.GetConnection(configuration.Password);
using var context = connection.Context;
var accounts = context.AccountsAsync().GetAwaiter().GetResult();
var result = new List <MoneyStateModel>();
foreach (var account in accounts)
{
var portfolio = context.PortfolioAsync(account.BrokerAccountId).GetAwaiter().GetResult();
result.AddRange(portfolio.Positions.Select(v => Money(account.BrokerAccountId + " " + v.Name, (double)(v.Balance * v.AveragePositionPrice.Value), v.AveragePositionPrice.Currency.ToString().ToUpper())));
var ccys = context.PortfolioCurrenciesAsync(account.BrokerAccountId).GetAwaiter().GetResult();
result.AddRange(ccys.Currencies.Select(v => Money(account.BrokerAccountId + " " + v.Currency.ToString().ToUpper(), (double)v.Balance, v.Currency.ToString().ToUpper())));
}
return(result);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
var result = new List <MoneyStateModel>();
driver.Navigate().GoToUrl("https://ru.investing.com/indices/us-spx-500");
var sps = GetElement(driver, By.Id("last_last")).Text;
result.Add(Money("SP500",
double.Parse(sps, new NumberFormatInfo()
{
NumberDecimalSeparator = ",", NumberGroupSeparator = "."
}),
CurrencyExtensions.USD));
driver.Navigate().GoToUrl("https://ru.investing.com/currencies/usd-rub");
sps = GetElement(driver, By.Id("last_last")).Text;
var usdRub = CurrencyExtensions.USD + "/" + CurrencyExtensions.RUB;
result.Add(Money(usdRub,
double.Parse(sps, new NumberFormatInfo()
{
NumberDecimalSeparator = ",", NumberGroupSeparator = "."
}), usdRub));
driver.Navigate().GoToUrl("https://ru.investing.com/currencies/eur-rub");
sps = GetElement(driver, By.Id("last_last")).Text;
var eurRub = CurrencyExtensions.EUR + "/" + CurrencyExtensions.RUB;
result.Add(Money(eurRub, double.Parse(sps,
new NumberFormatInfo()
{
NumberDecimalSeparator = ",", NumberGroupSeparator = "."
}), eurRub));
return(result);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
var result = new List <MoneyStateModel>();
driver.Navigate().GoToUrl("https://eodhistoricaldata.com/financial-summary/GSPC.INDX");
WaitForPageLoad(driver);
var moneyStateModel = ParseMoney("SP500", driver);
result.Add(moneyStateModel);
foreach (var item in CurrencyExtensions.KnownCurrencies.Where(v => v != CurrencyExtensions.RUB))
{
var url = $"https://eodhistoricaldata.com/financial-summary/{item.ToUpper()}RUB.FOREX";
driver.Navigate().GoToUrl(url);
Logger.LogInformation($"Scraping {url}");
var itemRub = item + "/" + CurrencyExtensions.RUB;
var msm = ParseMoney(itemRub, driver);
result.Add(msm);
}
return(result);
}
public IEnumerable <PaymentModel> ScrapeStatement(ScraperConfigurationModel configuration, Chrome chromeDriver,
DateTime startFrom)
{
if (startFrom < DateTime.Now.AddYears(-2).AddDays(1))
{
startFrom = DateTime.Now.AddYears(-2).AddDays(1);
}
var driver = chromeDriver.Driver;
Login(configuration, driver);
var existingLinks = driver.FindElementsByPartialLinkText("История операций").ToList();
var link1 = (RemoteWebElement)driver.FindElementByLinkText("Счета");
driver.Mouse.MouseMove(link1.Coordinates, 1, 1);
Thread.Sleep(500);
existingLinks = driver.FindElementsByPartialLinkText("История операций").Except(existingLinks).ToList();
existingLinks.Single().Click();
Thread.Sleep(2000);
var selectBtn = driver.FindElementById("pt1:soc1::button");
var accountsChooser = driver.FindElementById("pt1:soc1::pop");
var accs = accountsChooser.FindElements(By.TagName("tr"));
var result = new List <PaymentModel>();
foreach (var acc in accs)
{
var tds = acc.FindElements(By.TagName("td"));
if (tds.Count < 4)
{
continue;
}
selectBtn.Click();
Thread.Sleep(2000);
tds[0].Click();
Thread.Sleep(2000);
var inputDate = driver.FindElementById("pt1:id1::fd");
inputDate.Click();
Thread.Sleep(2000);
driver.Keyboard.SendKeys(Enumerable.Repeat(Keys.Delete, 20).Join(""));
Thread.Sleep(500);
driver.Keyboard.SendKeys(Enumerable.Repeat(Keys.Backspace, 20).Join(""));
Thread.Sleep(500);
driver.Keyboard.SendKeys(Enumerable.Repeat(Keys.Delete, 20).Join(""));
Thread.Sleep(500);
driver.Keyboard.SendKeys(Enumerable.Repeat(Keys.Backspace, 20).Join(""));
Thread.Sleep(500);
driver.Keyboard.SendKeys(startFrom.ToString("ddMMyyyy"));
var submit = driver.FindElementById("pt1:showButton::button");
submit.Click();
Thread.Sleep(2000);
var csv = driver.FindElementById("pt1:downloadCSVLink");
csv.Click();
int waited = 0;
while (chromeDriver.GetDownloads().Count < 1 && waited < 300)
{
Thread.Sleep(1000);
waited++;
}
var files = chromeDriver.GetDownloads();
if (files.Count == 1)
{
var csvFile = files.First();
var csvContent = File.ReadAllLines(csvFile.FullName, Encoding.GetEncoding(1251)).Skip(1).Select(v => new AlphaStatement(v)).ToList();
var payments = csvContent.Select(v =>
Statement(v.Date, v.AccountName, v.What, v.Outcome - v.Income, v.Ccy, v.Reference)).ToList();
var holdPayments = payments.Where(v => v.StatementReference == "HOLD").ToList();
payments = payments.Except(holdPayments).ToList();
result.AddRange(payments);
csvFile.Delete();
}
chromeDriver.CleanupDownloads();
}
return(result);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
driver.Navigate().GoToUrl("https://cabinet.moduldengi.ru/#/");
var auth = GetElement(driver, By.ClassName("auth"));
var phone = auth.FindElement(By.TagName("input"));
phone.Click();
foreach (var ch in configuration.Login)
{
chrome.SendKeys(ch.ToString());
WaitForPageLoad(driver);
}
chrome.SendKeys(Keys.Enter);
WaitForPageLoad(driver, 5);
var inputs = auth.FindElements(By.TagName("input"));
var pass = inputs.First(v => v.GetAttribute("id") != phone.GetAttribute("id"));
pass.Click();
chrome.SendKeys(configuration.Password);
chrome.SendKeys(Keys.Return);
WaitForPageLoad(driver, 5);
var row = GetElement(driver, By.ClassName("balances-row"));
var cells = row.FindElements(By.ClassName("balances-item"));
var result = new List <MoneyStateModel>();
foreach (var cell in cells)
{
var label = cell.FindElement(By.ClassName("property")).Text;
var value = cell.FindElement(By.ClassName("value")).Text;
var cleanValue = ParseDouble(value);
result.Add(Money(label, cleanValue, CurrencyExtensions.RUB));
}
var investmentsSection = GetElement(driver, By.ClassName("investments"));
while (investmentsSection.Text.ToLower().Contains("загрузка"))
{
WaitForPageLoad(driver);
}
var blockMenu = investmentsSection.FindElement(By.ClassName("block-menu"));
var links = blockMenu.FindElements(By.TagName("a"));
var badOnesButton = links.First(v => v.Text.ToLower() == "просроченные");
badOnesButton.Click();
WaitForPageLoad(driver);
var badProjects = investmentsSection.FindElements(By.ClassName("project-item-wrapper"));
var titles = badProjects.Select(v => v.FindElement(By.ClassName("project-table"))).ToList();
var spans = titles.Select(v =>
{
var rows = v.FindElements(By.TagName("tr")).Select(s => s.FindElements(By.TagName("td")).ToList()).ToList();
var parsed = rows.Select(s =>
{
var key = s[0].Text;
var value = s[1];
var spanValue = ParseDouble(value.FindElement(By.TagName("span")).Text);
return(new
{
key,
spanValue
});
}).ToList();
var inv = parsed.First(s => s.key.ToLower().Contains("инвестиция")).spanValue;
var ok = parsed.First(s => s.key.ToLower().Contains("погашено")).spanValue;
return(inv - ok);
}).ToList();
result.Add(Money("Просрочки", spans.Sum(), CurrencyExtensions.RUB));
return(result);
}
public override IList <PaymentModel> ScrapeStatement(ScraperConfigurationModel configuration, Chrome chrome, DateTime startFrom)
{
var driver = chrome.Driver;
Login(configuration, chrome);
driver.Navigate().GoToUrl(@"https://online.raiffeisen.ru/#/history/statement");
var accounts = GetElements(driver, By.TagName("c-select-option-account"));
var link = GetElements(driver, By.TagName("a")).First(v => v.GetAttribute("href")?.Contains("/transaction.ofx?") == true);
var linkText = link.GetAttribute("href");
var build = new Uri(linkText);
var result = new List <PaymentModel>();
var urlFormat = @"https://online.raiffeisen.ru/rest/account/{accountId}/transaction.ofx?from={from}&to={to}&sort=date&order=desc&access_token={token}";
var originalQuery = QueryHelpers.ParseQuery(build.Query);
var accessToken = originalQuery["access_token"].First();
var accountDetails = accounts.Select(v =>
{
var id = v.FindElement(By.TagName("div")).GetAttribute("data-account-id");
var textElement = v.FindElement(By.TagName("account-logo")).FindElement(By.XPath(".."));
var name = textElement.GetAttribute("textContent").Trim();
return(id, name);
}).Distinct().ToList();
Logger.LogInformation($"Found {accountDetails.Count} Raiffeisen accounts");
foreach (var account in accountDetails)
{
var accountId = account.id;
var accountName = account.name;
var url = urlFormat.Replace("{accountId}", accountId)
.Replace("{from}", startFrom.ToString("yyyy-MM-ddTHH:mm"))
.Replace("{to}", DateTime.Now.ToString("yyyy-MM-ddTHH:mm"))
.Replace("{token}", accessToken);
// Raiffeisen sometimes doesn't return all payments on first call
for (int i = 0; i < 3; i++)
{
driver.Navigate().GoToUrl(url);
Logger.LogInformation($"Getting statement for {account.name} at {url}, attempt {i}");
int waited = 0;
while (chrome.GetDownloads().Count < 1 && waited < 300)
{
WaitForPageLoad(driver);
waited++;
}
Thread.Sleep(10000);
var files = chrome.GetDownloads();
if (files.Count == 1)
{
var ofxFile = files.First();
var doc = File.ReadAllText(ofxFile.FullName);
var xdoc = XDocument.Parse(doc);
var statements =
xdoc.XPathSelectElements("OFX/BANKMSGSRSV1/STMTTRNRS/STMTRS/BANKTRANLIST/STMTTRN");
var ccyNode = xdoc.XPathSelectElement("OFX/BANKMSGSRSV1/STMTTRNRS/STMTRS/CURDEF");
var ccy = ccyNode.Value;
var payments = new List <PaymentModel>();
foreach (var st in statements)
{
var timeStr = st.Element("DTPOSTED").Value;
var time = DateTime.ParseExact(timeStr, "yyyyMMddhhmmss", CultureInfo.CurrentCulture);
var amountStr = st.Element("TRNAMT").Value;
var amount = double.Parse(amountStr,
new NumberFormatInfo {
NumberDecimalSeparator = "."
});
var name = st.Element("MEMO").Value;
var id = timeStr + amountStr + name;
int counter = 0;
while (payments.Any(j => j.StatementReference == id))
{
id = timeStr + amountStr + name + counter++;
}
var kind = amount < 0 ? PaymentKind.Expense : PaymentKind.Income;
var stmt = Statement(time, accountName, name, amount, kind, ccy, id);
payments.Add(stmt);
}
Logger.LogInformation($"Got {payments.Count} payments from {url}, attempt {i}");
result.AddRange(payments);
ofxFile.Delete();
}
}
}
var oldCount = result.Count;
result = result.GroupBy(v => v.StatementReference).Select(v => v.First()).ToList();
Logger.LogInformation($"Deduplicated payments {oldCount} -> {result.Count}");
return(result);
}
