public static List <OberloOrder> ScrapeOberloOrders(IMyConfiguration configuration, DateTime from, DateTime to)
{
var oberloOrders = new List <OberloOrder>();
string userDataDir = configuration.GetValue("UserDataDir");
string oberloUsername = configuration.GetValue("OberloUsername");
string oberloPassword = configuration.GetValue("OberloPassword");
string chromeDriverExePath = configuration.GetValue("ChromeDriverExePath");
var driver = Utils.GetChromeWebDriver(userDataDir, chromeDriverExePath);
// https://app.oberlo.com/orders?from=2017-01-01&to=2017-12-31&page=1
int page = 1;
string url = string.Format("https://app.oberlo.com/orders?from={0:yyyy-MM-dd}&to={1:yyyy-MM-dd}&page={2}", from, to, page);
driver.Navigate().GoToUrl(url);
var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
var ready = wait.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
// login if login form is present
if (SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@name='email']")) &&
SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@name='password']")))
{
IWebElement username = driver.FindElement(By.XPath("//input[@name='email']"));
IWebElement password = driver.FindElement(By.XPath("//input[@name='password']"));
username.Clear();
username.SendKeys(oberloUsername);
// if the above crash, it might very well be Chrome and ChromeDriver incompatibility
// please update the ChromeDriver to fit the Chrome version used.
password.Clear();
password.SendKeys(oberloPassword);
// use password field to submit form
password.Submit();
var wait2 = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
var ready2 = wait2.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
}
IJavaScriptExecutor js = driver as IJavaScriptExecutor;
var json = js.ExecuteScript("return window.App.payload.orders;");
// convert to json dynamic object
string jsonString = JsonConvert.SerializeObject(json);
dynamic jsonD = JsonConvert.DeserializeObject(jsonString);
// https://app.oberlo.com/orders?from=2017-01-01&to=2017-12-31&page=1
// identify how many pages on order page
// current_page
// last_page
// per_page
// data (System.Collections.ObjectModel)
int current_page = jsonD.current_page;
int last_page = jsonD.last_page;
int per_page = jsonD.per_page;
// process orders on page
var orders = jsonD.data;
foreach (var order in orders)
{
// order_name
// processed_at
// total_price
// shipping_name
// shipping_zip
// shipping_city
// shipping_address1
// orderitems
var orderName = order.order_name;
var processedAt = order.processed_at;
var totalPrice = order.total_price;
var financialStatus = order.financial_status;
var fulfillmentStatus = order.fulfillment_status;
var shippingName = order.shipping_name;
var shippingZip = order.shipping_zip;
var shippingCity = order.shipping_city;
var shippingAddress1 = order.shipping_address1;
var shippingAddress2 = order.shipping_address2;
var orderNote = order.local_note;
var orderitems = order.orderitems;
foreach (var orderitem in orderitems)
{
var aliOrderNumber = orderitem.ali_order_no;
var SKU = orderitem.sku;
var supplier = orderitem.supplier_name;
var productName = orderitem.title;
var variant = orderitem.variant_title;
var cost = orderitem.cost;
var quantity = orderitem.quantity;
var price = orderitem.price;
string trackingNumber = "";
foreach (var fulfillment in orderitem.fulfillments)
{
if (trackingNumber.Equals(""))
{
trackingNumber = fulfillment.tracking_number;
}
else
{
trackingNumber += ", " + fulfillment.tracking_number;
}
}
var oberloOrder = new OberloOrder();
oberloOrder.OrderNumber = orderName;
oberloOrder.CreatedDate = processedAt;
oberloOrder.FinancialStatus = financialStatus;
oberloOrder.FulfillmentStatus = fulfillmentStatus;
oberloOrder.Supplier = supplier;
oberloOrder.SKU = SKU;
oberloOrder.ProductName = productName;
oberloOrder.Variant = variant;
oberloOrder.Quantity = quantity;
oberloOrder.ProductPrice = price;
oberloOrder.TrackingNumber = trackingNumber;
oberloOrder.AliOrderNumber = aliOrderNumber;
oberloOrder.CustomerName = shippingName;
oberloOrder.CustomerAddress = shippingAddress1;
oberloOrder.CustomerAddress2 = shippingAddress2;
oberloOrder.CustomerCity = shippingCity;
oberloOrder.CustomerZip = shippingZip;
oberloOrder.OrderNote = orderNote;
//oberloOrder.OrderState = orderState;
oberloOrder.TotalPrice = totalPrice;
oberloOrder.Cost = cost;
oberloOrders.Add(oberloOrder);
}
}
// and process the rest of the pages
for (int i = current_page + 1; i <= last_page; i++)
{
ScrapeOberloOrders(driver, oberloOrders, from, to, i);
}
driver.Close();
return(oberloOrders);
}
public static string DownloadBankStatement(IMyConfiguration configuration, DateTime from, DateTime to)
{
string cacheDir = configuration.GetValue("CacheDir");
string userDataDir = configuration.GetValue("UserDataDir");
string sbankenMobilePhone = configuration.GetValue("SBankenMobilePhone");
string sbankenBirthDate = configuration.GetValue("SBankenBirthDate");
string cacheFileNamePrefix = configuration.GetValue("SBankenAccountNumber");
string sbankenAccountId = configuration.GetValue("SBankenAccountId");
string downloadFolderPath = Environment.GetEnvironmentVariable("USERPROFILE") + @"\Downloads\";
string chromeDriverExePath = configuration.GetValue("ChromeDriverExePath");
var driver = Utils.GetChromeWebDriver(userDataDir, chromeDriverExePath);
driver.Navigate().GoToUrl("https://secure.sbanken.no/Authentication/BankIdMobile");
var waitLoginPage = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitLoginPage.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
// login if login form is present
if (SeleniumUtils.IsElementPresent(driver, By.Id("MobilePhone")))
{
// https://secure.sbanken.no/Authentication/BankIDMobile
// input id MobilePhone - Mobilnummer (8 siffer)
// input it BirthDate - Fødselsdato (ddmmåå)
// submit value = Neste
// login if login form is present
if (SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@id='MobilePhone']")) &&
SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@id='BirthDate']")))
{
IWebElement mobilePhone = driver.FindElement(By.XPath("//input[@id='MobilePhone']"));
IWebElement birthDate = driver.FindElement(By.XPath("//input[@id='BirthDate']"));
mobilePhone.Clear();
mobilePhone.SendKeys(sbankenMobilePhone);
birthDate.Clear();
birthDate.SendKeys(sbankenBirthDate);
// use birth date field to submit form
birthDate.Submit();
var waitLoginIFrame = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitLoginIFrame.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
}
}
try
{
var waitMainPage = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitMainPage.Until(SeleniumExtras.WaitHelpers.ExpectedConditions.UrlToBe("https://secure.sbanken.no/Home/Overview/Full#/"));
}
catch (WebDriverTimeoutException)
{
Console.WriteLine("Timeout - Logged in to Skandiabanken too late. Stopping.");
return(null);
}
// download account statement
string accountStatementDownload = string.Format("https://secure.sbanken.no/Home/AccountStatement/ViewExcel?AccountId={0}&CustomFromDate={1:dd.MM.yyyy}&CustomToDate={2:dd.MM.yyyy}&FromDate=CustomPeriod&Incoming=", sbankenAccountId, from, to);
driver.Navigate().GoToUrl(accountStatementDownload);
var waitExcel = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitExcel.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
string accountStatementFileName = string.Format("{0}_{1:yyyy_MM_dd}-{2:yyyy_MM_dd}.xlsx", cacheFileNamePrefix, from, to);
string accountStatementDownloadFilePath = Path.Combine(downloadFolderPath, accountStatementFileName);
// wait until file has downloaded
for (var i = 0; i < 30; i++)
{
if (File.Exists(accountStatementDownloadFilePath))
{
break;
}
Thread.Sleep(1000);
}
var length = new FileInfo(accountStatementDownloadFilePath).Length;
for (var i = 0; i < 30; i++)
{
Thread.Sleep(1000);
var newLength = new FileInfo(accountStatementDownloadFilePath).Length;
if (newLength == length && length != 0)
{
break;
}
length = newLength;
}
driver.Close();
// determine path
Console.Out.WriteLine("Successfully downloaded skandiabanken account statement excel file {0}", accountStatementDownloadFilePath);
// moving file to right place
string accountStatementDestinationPath = Path.Combine(cacheDir, accountStatementFileName);
// To copy a folder's contents to a new location:
// Create a new target folder, if necessary.
if (!Directory.Exists(cacheDir))
{
Directory.CreateDirectory(cacheDir);
}
// Move file to another location
File.Move(accountStatementDownloadFilePath, accountStatementDestinationPath);
return(accountStatementDestinationPath);
}
public static List <AliExpressOrder> ScrapeAliExpressOrders(IMyConfiguration configuration, DateTime from)
{
string userDataDir = configuration.GetValue("UserDataDir");
string cacheDir = configuration.GetValue("CacheDir");
string aliExpressUsername = configuration.GetValue("AliExpressUsername");
string aliExpressPassword = configuration.GetValue("AliExpressPassword");
string chromeDriverExePath = configuration.GetValue("ChromeDriverExePath");
var aliExpressOrders = new List <AliExpressOrder>();
// http://blog.hanxiaogang.com/2017-07-29-aliexpress/
var driver = Utils.GetChromeWebDriver(userDataDir, chromeDriverExePath);
driver.Navigate().GoToUrl("https://login.aliexpress.com");
var waitLoginPage = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitLoginPage.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
// login if login form is present
// https://philipcaande.wordpress.com/2016/01/17/iframe-in-selenium-c/
if (SeleniumUtils.IsElementPresent(driver, By.Id("alibaba-login-box")))
{
// switch to login iframe
IWebElement iframe = driver.FindElement(By.Id("alibaba-login-box"));
driver.SwitchTo().Frame(iframe);
// login if login form is present
if (SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@id='fm-login-id']")) &&
SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@id='fm-login-password']")))
{
IWebElement username = driver.FindElement(By.XPath("//input[@id='fm-login-id']"));
IWebElement password = driver.FindElement(By.XPath("//input[@id='fm-login-password']"));
username.Clear();
username.SendKeys(aliExpressUsername);
password.Clear();
password.SendKeys(aliExpressPassword);
// use password field to submit form
password.Submit();
var waitLoginIFrame = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitLoginIFrame.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
}
// switch back to main frame
driver.SwitchTo().DefaultContent();
}
try
{
var waitMainPage = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitMainPage.Until(SeleniumExtras.WaitHelpers.ExpectedConditions.UrlToBe("https://www.aliexpress.com/"));
}
catch (WebDriverTimeoutException)
{
Console.WriteLine("Timeout - Logged in to AliExpress to late. Stopping.");
return(aliExpressOrders);
}
// go to order list
driver.Navigate().GoToUrl("https://trade.aliexpress.com/orderList.htm");
// identify how many pages on order page (1/20)
var tuple = GetAliExpressOrderPageNumber(driver);
int curPage = tuple.Item1;
int numPages = tuple.Item2;
Console.WriteLine("Found {0} Pages", numPages);
// scrape one and one page
for (int i = 1; i <= numPages; i++)
{
// if this method returns false, it means we have reached the from date
if (!ScrapeAliExpressOrderPage(aliExpressOrders, driver, i, from))
{
break;
}
}
driver.Close();
return(aliExpressOrders);
}
