public IEnumerable <PaymentModel> ScrapeStatement(ScraperConfigurationModel configuration, Chrome chromeDriver,
DateTime startFrom)
{
if (startFrom < DateTime.Now.AddYears(-2).AddDays(1))
{
startFrom = DateTime.Now.AddYears(-2).AddDays(1);
}
var driver = chromeDriver.Driver;
Login(configuration, driver);
var existingLinks = driver.FindElementsByPartialLinkText("История операций").ToList();
var link1 = (RemoteWebElement)driver.FindElementByLinkText("Счета");
driver.Mouse.MouseMove(link1.Coordinates, 1, 1);
Thread.Sleep(500);
existingLinks = driver.FindElementsByPartialLinkText("История операций").Except(existingLinks).ToList();
existingLinks.Single().Click();
Thread.Sleep(2000);
var selectBtn = driver.FindElementById("pt1:soc1::button");
var accountsChooser = driver.FindElementById("pt1:soc1::pop");
var accs = accountsChooser.FindElements(By.TagName("tr"));
var result = new List <PaymentModel>();
foreach (var acc in accs)
{
var tds = acc.FindElements(By.TagName("td"));
if (tds.Count < 4)
{
continue;
}
selectBtn.Click();
Thread.Sleep(2000);
tds[0].Click();
Thread.Sleep(2000);
var inputDate = driver.FindElementById("pt1:id1::fd");
inputDate.Click();
Thread.Sleep(2000);
driver.Keyboard.SendKeys(Enumerable.Repeat(Keys.Delete, 20).Join(""));
Thread.Sleep(500);
driver.Keyboard.SendKeys(Enumerable.Repeat(Keys.Backspace, 20).Join(""));
Thread.Sleep(500);
driver.Keyboard.SendKeys(Enumerable.Repeat(Keys.Delete, 20).Join(""));
Thread.Sleep(500);
driver.Keyboard.SendKeys(Enumerable.Repeat(Keys.Backspace, 20).Join(""));
Thread.Sleep(500);
driver.Keyboard.SendKeys(startFrom.ToString("ddMMyyyy"));
var submit = driver.FindElementById("pt1:showButton::button");
submit.Click();
Thread.Sleep(2000);
var csv = driver.FindElementById("pt1:downloadCSVLink");
csv.Click();
int waited = 0;
while (chromeDriver.GetDownloads().Count < 1 && waited < 300)
{
Thread.Sleep(1000);
waited++;
}
var files = chromeDriver.GetDownloads();
if (files.Count == 1)
{
var csvFile = files.First();
var csvContent = File.ReadAllLines(csvFile.FullName, Encoding.GetEncoding(1251)).Skip(1).Select(v => new AlphaStatement(v)).ToList();
var payments = csvContent.Select(v =>
Statement(v.Date, v.AccountName, v.What, v.Outcome - v.Income, v.Ccy, v.Reference)).ToList();
var holdPayments = payments.Where(v => v.StatementReference == "HOLD").ToList();
payments = payments.Except(holdPayments).ToList();
result.AddRange(payments);
csvFile.Delete();
}
chromeDriver.CleanupDownloads();
}
return(result);
}
public override IList <PaymentModel> ScrapeStatement(ScraperConfigurationModel configuration, Chrome chrome, DateTime startFrom)
{
var driver = chrome.Driver;
Login(configuration, chrome);
driver.Navigate().GoToUrl(@"https://online.raiffeisen.ru/#/history/statement");
var accounts = GetElements(driver, By.TagName("c-select-option-account"));
var link = GetElements(driver, By.TagName("a")).First(v => v.GetAttribute("href")?.Contains("/transaction.ofx?") == true);
var linkText = link.GetAttribute("href");
var build = new Uri(linkText);
var result = new List <PaymentModel>();
var urlFormat = @"https://online.raiffeisen.ru/rest/account/{accountId}/transaction.ofx?from={from}&to={to}&sort=date&order=desc&access_token={token}";
var originalQuery = QueryHelpers.ParseQuery(build.Query);
var accessToken = originalQuery["access_token"].First();
var accountDetails = accounts.Select(v =>
{
var id = v.FindElement(By.TagName("div")).GetAttribute("data-account-id");
var textElement = v.FindElement(By.TagName("account-logo")).FindElement(By.XPath(".."));
var name = textElement.GetAttribute("textContent").Trim();
return(id, name);
}).Distinct().ToList();
Logger.LogInformation($"Found {accountDetails.Count} Raiffeisen accounts");
foreach (var account in accountDetails)
{
var accountId = account.id;
var accountName = account.name;
var url = urlFormat.Replace("{accountId}", accountId)
.Replace("{from}", startFrom.ToString("yyyy-MM-ddTHH:mm"))
.Replace("{to}", DateTime.Now.ToString("yyyy-MM-ddTHH:mm"))
.Replace("{token}", accessToken);
// Raiffeisen sometimes doesn't return all payments on first call
for (int i = 0; i < 3; i++)
{
driver.Navigate().GoToUrl(url);
Logger.LogInformation($"Getting statement for {account.name} at {url}, attempt {i}");
int waited = 0;
while (chrome.GetDownloads().Count < 1 && waited < 300)
{
WaitForPageLoad(driver);
waited++;
}
Thread.Sleep(10000);
var files = chrome.GetDownloads();
if (files.Count == 1)
{
var ofxFile = files.First();
var doc = File.ReadAllText(ofxFile.FullName);
var xdoc = XDocument.Parse(doc);
var statements =
xdoc.XPathSelectElements("OFX/BANKMSGSRSV1/STMTTRNRS/STMTRS/BANKTRANLIST/STMTTRN");
var ccyNode = xdoc.XPathSelectElement("OFX/BANKMSGSRSV1/STMTTRNRS/STMTRS/CURDEF");
var ccy = ccyNode.Value;
var payments = new List <PaymentModel>();
foreach (var st in statements)
{
var timeStr = st.Element("DTPOSTED").Value;
var time = DateTime.ParseExact(timeStr, "yyyyMMddhhmmss", CultureInfo.CurrentCulture);
var amountStr = st.Element("TRNAMT").Value;
var amount = double.Parse(amountStr,
new NumberFormatInfo {
NumberDecimalSeparator = "."
});
var name = st.Element("MEMO").Value;
var id = timeStr + amountStr + name;
int counter = 0;
while (payments.Any(j => j.StatementReference == id))
{
id = timeStr + amountStr + name + counter++;
}
var kind = amount < 0 ? PaymentKind.Expense : PaymentKind.Income;
var stmt = Statement(time, accountName, name, amount, kind, ccy, id);
payments.Add(stmt);
}
Logger.LogInformation($"Got {payments.Count} payments from {url}, attempt {i}");
result.AddRange(payments);
ofxFile.Delete();
}
}
}
var oldCount = result.Count;
result = result.GroupBy(v => v.StatementReference).Select(v => v.First()).ToList();
Logger.LogInformation($"Deduplicated payments {oldCount} -> {result.Count}");
return(result);
}
public override IList <MoneyStateModel> Scrape(ScraperConfigurationModel configuration, Chrome chrome)
{
var driver = chrome.Driver;
driver.Navigate().GoToUrl("https://lk.alfadirect.ru/");
var fields = GetElements(driver, By.TagName("input")).ToList();
var name = fields[0];
var pass = fields[1];
name.Click();
chrome.SendKeys(configuration.Login);
pass.Click();
chrome.SendKeys(configuration.Password);
chrome.SendKeys(Keys.Return);
WaitForPageLoad(driver);
driver.Navigate().GoToUrl("https://lk.alfadirect.ru/reports/MyPortfolio");
WaitForPageLoad(driver);
var lnk = GetElement(driver, By.LinkText("Просмотреть"));
var link = lnk.GetAttribute("href");
link = link.Replace("HTML", "XML");
driver.Navigate().GoToUrl(link);
int waited = 0;
while (chrome.GetDownloads().Count < 1 && waited < 300)
{
WaitForPageLoad(driver);
waited++;
}
var file = chrome.GetDownloads()[0].FullName;
var contents = File.ReadAllText(file);
var xDoc = XDocument.Parse(contents);
var el = xDoc.Root;
el = el.Element(XName.Get("Financial_results", "MyPortfolio"));
el = el.Element(XName.Get("Report", "MyPortfolio"));
var positions = el.Descendants(XName.Get("Details", "MyPortfolio")).ToList();
var result = new List <MoneyStateModel>();
foreach (var item in positions)
{
var ccy = item.Attribute("code_curr")?.Value;
var activeType = item.Attribute("active_type")?.Value;
var activeName = item.Attributes().FirstOrDefault(s =>
s.Name.LocalName.StartsWith("p_name") && !string.IsNullOrWhiteSpace(s.Value))?.Value;
var activeCurrentPrice = item.Attribute("CostOpenPosEnd8")?.Value;
if (!double.TryParse(activeCurrentPrice, NumberStyles.Any, new NumberFormatInfo {
NumberDecimalSeparator = "."
}, out var amount))
{
continue;
}
result.Add(Money(activeName ?? (activeType + " " + ccy), amount, ccy));
}
var totals = result.GroupBy(v => v.Ccy).Select(s => Money("Итого " + s.Key, s.Sum(v => v.Amount), s.Key)).ToList();
result.AddRange(totals);
return(result);
}
public override IList <PaymentModel> ScrapeStatement(ScraperConfigurationModel configuration, Chrome chrome, DateTime startFrom)
{
DoLogin(configuration, chrome);
var driver = chrome.Driver;
var btns = GetElements(driver, By.TagName("button"));
var rightBtn = btns.First(s => s.Text.ToLower().Contains("выписка"));
rightBtn.Click();
var form = GetElement(driver, By.ClassName("address_form"));
var switches = form.FindElements(By.ClassName("switch_item"));
var htmlSwitch = switches.First(s => s.Text == "1C");
htmlSwitch.Click();
var rows = form.FindElements(By.ClassName("form_group"));
var periodRow = rows.First(v => v.Text.ToLower().Contains("за период:"));
var periodButton = periodRow.FindElement(By.TagName("button"));
periodButton.Click();
var dateSelector = GetElement(driver, By.ClassName("filter_date_value"));
var inputs = dateSelector.FindElements(By.TagName("input"));
var first = inputs.First();
first.Click();
first.SendKeys(Keys.Control + "a");
first.SendKeys(Keys.Delete);
first.SendKeys(startFrom.ToString("dd.MM.yyyy"));
first.SendKeys(Keys.Enter);
var dateSelectorApply = dateSelector.FindElement(By.TagName("button"));
dateSelectorApply.Click();
var buttons = form.FindElements(By.TagName("button"));
var dlButton = buttons.First(s => s.Text.ToLower().Contains("получить"));
chrome.CleanupDownloads();
dlButton.Click();
var waited = 0;
while (chrome.GetDownloads().Count == 0 && waited < 300)
{
WaitForPageLoad(driver);
waited++;
}
var dlItem = chrome.GetDownloads().First().FullName;
var docs = ParseOdinAssFile(dlItem);
var config = docs.Single(v => !v.ContainsKey("СекцияДокумент"));
var account = config["РасчСчет"];
var goodDocs = docs.Where(v => v.ContainsKey("СекцияДокумент"));
var statements = goodDocs.Select(v =>
{
var whenString = v["Дата"];
var when = DateTime.ParseExact(whenString, "dd.MM.yyyy", CultureInfo.InvariantCulture);
var what = v["НазначениеПлатежа"];
var amountText = v["Сумма"];
var amount = double.Parse(amountText, new NumberFormatInfo()
{
NumberDecimalSeparator = "."
});
var isIncome = v["ПолучательСчет"] == account;
var kind = isIncome ? PaymentKind.Income : PaymentKind.Expense;
var ccy = "RUB";
var reference = v["Номер"];
return(Statement(when, account, what, amount, kind, ccy, reference));
}).ToList();
return(statements);
}
public override IList <PaymentModel> ScrapeStatement(ScraperConfigurationModel configuration, Chrome chrome, DateTime startFrom)
{
var driver = chrome.Driver;
Login(configuration, chrome);
driver.Navigate().GoToUrl(@"https://online.raiffeisen.ru/#/history/statement");
var accounts = GetElements(driver, By.TagName("c-select-option-account"));
var link = GetElements(driver, By.TagName("a")).First(v => v.GetAttribute("href")?.Contains("/transaction.ofx?") == true);
var linkText = link.GetAttribute("href");
var build = new Uri(linkText);
var result = new List <PaymentModel>();
var urlFormat = @"https://online.raiffeisen.ru/rest/account/{accountId}/transaction.ofx?from={from}&to={to}&sort=date&order=desc&access_token={token}";
var originalQuery = QueryHelpers.ParseQuery(build.Query);
var accessToken = originalQuery["access_token"].First();
var accountDetails = accounts.Select(v =>
{
var id = v.FindElement(By.TagName("div")).GetAttribute("data-account-id");
var textElement = v.FindElement(By.TagName("account-logo")).FindElement(By.XPath(".."));
var name = textElement.GetAttribute("textContent").Trim();
return(id, name);
}).Distinct().ToList();
Logger.LogInformation($"Found {accountDetails.Count} Raiffeisen accounts");
foreach (var account in accountDetails)
{
var accountId = account.id;
var accountName = account.name;
var url = urlFormat.Replace("{accountId}", accountId)
.Replace("{from}", startFrom.ToString("yyyy-MM-ddTHH:mm"))
.Replace("{to}", DateTime.Now.ToString("yyyy-MM-ddTHH:mm"))
.Replace("{token}", accessToken);
// Raiffeisen sometimes doesn't return all payments on first call
for (int i = 0; i < 3; i++)
{
driver.Navigate().GoToUrl(url);
Logger.LogInformation($"Getting statement for {account.name} at {url}, attempt {i}");
int waited = 0;
while (chrome.GetDownloads().Count < 1 && waited < 300)
{
WaitForPageLoad(driver);
waited++;
}
Thread.Sleep(10000);
var files = chrome.GetDownloads();
if (files.Count == 1)
{
var ofxFile = files.First();
var doc = File.ReadAllText(ofxFile.FullName);
var xdoc = XDocument.Parse(doc);
var payments = ParseOfx(xdoc, accountName);
Logger.LogInformation($"Got {payments.Count} payments from {url}, attempt {i}");
result.AddRange(payments);
ofxFile.Delete();
}
}
}
var oldCount = result.Count;
result = result.GroupBy(v => v.StatementReference).Select(v => v.First()).ToList();
Logger.LogInformation($"Deduplicated payments {oldCount} -> {result.Count}");
return(result);
}
public override IList <PaymentModel> ScrapeStatement(ScraperConfigurationModel configuration, Chrome chromeDriver,
DateTime startFrom)
{
if (startFrom < DateTime.Now.AddYears(-2).AddDays(1))
{
startFrom = DateTime.Now.AddYears(-2).AddDays(1);
}
var driver = chromeDriver.Driver;
Login(configuration, chromeDriver);
var unixTime = new DateTime(1970, 1, 1, 0, 0, 0, 0);
var startTime = (long)(startFrom - unixTime).TotalMilliseconds;
var endTime = (long)(DateTime.UtcNow - unixTime).TotalMilliseconds;
var cardId = 0;
var sessionId = driver.Manage().Cookies.GetCookieNamed("psid").Value;
var links = GetElements(driver, By.TagName("a")).ToList();
var matchingLinks = links.Where(v => v.GetProperty("href").Contains("/events/account/")).ToList();
var result = new List <PaymentModel>();
var accountRegex = new Regex(@"\/events\/account\/([^\/]+)\/(?<accountId>[0-9]+)");
foreach (var link in matchingLinks)
{
var href = link.GetAttribute("href");
if (href == null)
{
continue;
}
var match = accountRegex.Match(href);
if (match.Success)
{
var children = link.FindElements(By.TagName("div"));
var name = children.First(v => v.GetAttribute("class").Contains("Item__name"));
var accountName = name.Text;
var accountId = match.Groups["accountId"].Value;
var url = $"https://www.tinkoff.ru/api/common/v1/export_operations/?format=ofx&sessionid={sessionId}&start={startTime}&end={endTime}&card={cardId}&account={accountId}";
driver.Navigate().GoToUrl(url);
int waited = 0;
while (chromeDriver.GetDownloads().Count < 1 && waited < 300)
{
WaitForPageLoad(driver);
waited++;
}
var file = chromeDriver.GetDownloads()[0].FullName;
var contents = File.ReadAllText(file);
var ofx = XDocument.Parse(contents);
var payments = ParseOfx(ofx, accountName);
result.AddRange(payments);
File.Delete(file);
}
}
return(result);
}