public static List <OberloOrder> ScrapeOberloOrders(IMyConfiguration configuration, DateTime from, DateTime to) { var oberloOrders = new List <OberloOrder>(); string userDataDir = configuration.GetValue("UserDataDir"); string oberloUsername = configuration.GetValue("OberloUsername"); string oberloPassword = configuration.GetValue("OberloPassword"); string chromeDriverExePath = configuration.GetValue("ChromeDriverExePath"); var driver = Utils.GetChromeWebDriver(userDataDir, chromeDriverExePath); // https://app.oberlo.com/orders?from=2017-01-01&to=2017-12-31&page=1 int page = 1; string url = string.Format("https://app.oberlo.com/orders?from={0:yyyy-MM-dd}&to={1:yyyy-MM-dd}&page={2}", from, to, page); driver.Navigate().GoToUrl(url); var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(30)); var ready = wait.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete")); // login if login form is present if (SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@name='email']")) && SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@name='password']"))) { IWebElement username = driver.FindElement(By.XPath("//input[@name='email']")); IWebElement password = driver.FindElement(By.XPath("//input[@name='password']")); username.Clear(); username.SendKeys(oberloUsername); // if the above crash, it might very well be Chrome and ChromeDriver incompatibility // please update the ChromeDriver to fit the Chrome version used. password.Clear(); password.SendKeys(oberloPassword); // use password field to submit form password.Submit(); var wait2 = new WebDriverWait(driver, TimeSpan.FromSeconds(30)); var ready2 = wait2.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete")); } IJavaScriptExecutor js = driver as IJavaScriptExecutor; var json = js.ExecuteScript("return window.App.payload.orders;"); // convert to json dynamic object string jsonString = JsonConvert.SerializeObject(json); dynamic jsonD = JsonConvert.DeserializeObject(jsonString); // https://app.oberlo.com/orders?from=2017-01-01&to=2017-12-31&page=1 // identify how many pages on order page // current_page // last_page // per_page // data (System.Collections.ObjectModel) int current_page = jsonD.current_page; int last_page = jsonD.last_page; int per_page = jsonD.per_page; // process orders on page var orders = jsonD.data; foreach (var order in orders) { // order_name // processed_at // total_price // shipping_name // shipping_zip // shipping_city // shipping_address1 // orderitems var orderName = order.order_name; var processedAt = order.processed_at; var totalPrice = order.total_price; var financialStatus = order.financial_status; var fulfillmentStatus = order.fulfillment_status; var shippingName = order.shipping_name; var shippingZip = order.shipping_zip; var shippingCity = order.shipping_city; var shippingAddress1 = order.shipping_address1; var shippingAddress2 = order.shipping_address2; var orderNote = order.local_note; var orderitems = order.orderitems; foreach (var orderitem in orderitems) { var aliOrderNumber = orderitem.ali_order_no; var SKU = orderitem.sku; var supplier = orderitem.supplier_name; var productName = orderitem.title; var variant = orderitem.variant_title; var cost = orderitem.cost; var quantity = orderitem.quantity; var price = orderitem.price; string trackingNumber = ""; foreach (var fulfillment in orderitem.fulfillments) { if (trackingNumber.Equals("")) { trackingNumber = fulfillment.tracking_number; } else { trackingNumber += ", " + fulfillment.tracking_number; } } var oberloOrder = new OberloOrder(); oberloOrder.OrderNumber = orderName; oberloOrder.CreatedDate = processedAt; oberloOrder.FinancialStatus = financialStatus; oberloOrder.FulfillmentStatus = fulfillmentStatus; oberloOrder.Supplier = supplier; oberloOrder.SKU = SKU; oberloOrder.ProductName = productName; oberloOrder.Variant = variant; oberloOrder.Quantity = quantity; oberloOrder.ProductPrice = price; oberloOrder.TrackingNumber = trackingNumber; oberloOrder.AliOrderNumber = aliOrderNumber; oberloOrder.CustomerName = shippingName; oberloOrder.CustomerAddress = shippingAddress1; oberloOrder.CustomerAddress2 = shippingAddress2; oberloOrder.CustomerCity = shippingCity; oberloOrder.CustomerZip = shippingZip; oberloOrder.OrderNote = orderNote; //oberloOrder.OrderState = orderState; oberloOrder.TotalPrice = totalPrice; oberloOrder.Cost = cost; oberloOrders.Add(oberloOrder); } } // and process the rest of the pages for (int i = current_page + 1; i <= last_page; i++) { ScrapeOberloOrders(driver, oberloOrders, from, to, i); } driver.Close(); return(oberloOrders); }
public static string DownloadBankStatement(IMyConfiguration configuration, DateTime from, DateTime to) { string cacheDir = configuration.GetValue("CacheDir"); string userDataDir = configuration.GetValue("UserDataDir"); string sbankenMobilePhone = configuration.GetValue("SBankenMobilePhone"); string sbankenBirthDate = configuration.GetValue("SBankenBirthDate"); string cacheFileNamePrefix = configuration.GetValue("SBankenAccountNumber"); string sbankenAccountId = configuration.GetValue("SBankenAccountId"); string downloadFolderPath = Environment.GetEnvironmentVariable("USERPROFILE") + @"\Downloads\"; string chromeDriverExePath = configuration.GetValue("ChromeDriverExePath"); var driver = Utils.GetChromeWebDriver(userDataDir, chromeDriverExePath); driver.Navigate().GoToUrl("https://secure.sbanken.no/Authentication/BankIdMobile"); var waitLoginPage = new WebDriverWait(driver, TimeSpan.FromSeconds(30)); waitLoginPage.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete")); // login if login form is present if (SeleniumUtils.IsElementPresent(driver, By.Id("MobilePhone"))) { // https://secure.sbanken.no/Authentication/BankIDMobile // input id MobilePhone - Mobilnummer (8 siffer) // input it BirthDate - Fødselsdato (ddmmåå) // submit value = Neste // login if login form is present if (SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@id='MobilePhone']")) && SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@id='BirthDate']"))) { IWebElement mobilePhone = driver.FindElement(By.XPath("//input[@id='MobilePhone']")); IWebElement birthDate = driver.FindElement(By.XPath("//input[@id='BirthDate']")); mobilePhone.Clear(); mobilePhone.SendKeys(sbankenMobilePhone); birthDate.Clear(); birthDate.SendKeys(sbankenBirthDate); // use birth date field to submit form birthDate.Submit(); var waitLoginIFrame = new WebDriverWait(driver, TimeSpan.FromSeconds(30)); waitLoginIFrame.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete")); } } try { var waitMainPage = new WebDriverWait(driver, TimeSpan.FromSeconds(30)); waitMainPage.Until(SeleniumExtras.WaitHelpers.ExpectedConditions.UrlToBe("https://secure.sbanken.no/Home/Overview/Full#/")); } catch (WebDriverTimeoutException) { Console.WriteLine("Timeout - Logged in to Skandiabanken too late. Stopping."); return(null); } // download account statement string accountStatementDownload = string.Format("https://secure.sbanken.no/Home/AccountStatement/ViewExcel?AccountId={0}&CustomFromDate={1:dd.MM.yyyy}&CustomToDate={2:dd.MM.yyyy}&FromDate=CustomPeriod&Incoming=", sbankenAccountId, from, to); driver.Navigate().GoToUrl(accountStatementDownload); var waitExcel = new WebDriverWait(driver, TimeSpan.FromSeconds(30)); waitExcel.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete")); string accountStatementFileName = string.Format("{0}_{1:yyyy_MM_dd}-{2:yyyy_MM_dd}.xlsx", cacheFileNamePrefix, from, to); string accountStatementDownloadFilePath = Path.Combine(downloadFolderPath, accountStatementFileName); // wait until file has downloaded for (var i = 0; i < 30; i++) { if (File.Exists(accountStatementDownloadFilePath)) { break; } Thread.Sleep(1000); } var length = new FileInfo(accountStatementDownloadFilePath).Length; for (var i = 0; i < 30; i++) { Thread.Sleep(1000); var newLength = new FileInfo(accountStatementDownloadFilePath).Length; if (newLength == length && length != 0) { break; } length = newLength; } driver.Close(); // determine path Console.Out.WriteLine("Successfully downloaded skandiabanken account statement excel file {0}", accountStatementDownloadFilePath); // moving file to right place string accountStatementDestinationPath = Path.Combine(cacheDir, accountStatementFileName); // To copy a folder's contents to a new location: // Create a new target folder, if necessary. if (!Directory.Exists(cacheDir)) { Directory.CreateDirectory(cacheDir); } // Move file to another location File.Move(accountStatementDownloadFilePath, accountStatementDestinationPath); return(accountStatementDestinationPath); }
public static List <AliExpressOrder> ScrapeAliExpressOrders(IMyConfiguration configuration, DateTime from) { string userDataDir = configuration.GetValue("UserDataDir"); string cacheDir = configuration.GetValue("CacheDir"); string aliExpressUsername = configuration.GetValue("AliExpressUsername"); string aliExpressPassword = configuration.GetValue("AliExpressPassword"); string chromeDriverExePath = configuration.GetValue("ChromeDriverExePath"); var aliExpressOrders = new List <AliExpressOrder>(); // http://blog.hanxiaogang.com/2017-07-29-aliexpress/ var driver = Utils.GetChromeWebDriver(userDataDir, chromeDriverExePath); driver.Navigate().GoToUrl("https://login.aliexpress.com"); var waitLoginPage = new WebDriverWait(driver, TimeSpan.FromSeconds(30)); waitLoginPage.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete")); // login if login form is present // https://philipcaande.wordpress.com/2016/01/17/iframe-in-selenium-c/ if (SeleniumUtils.IsElementPresent(driver, By.Id("alibaba-login-box"))) { // switch to login iframe IWebElement iframe = driver.FindElement(By.Id("alibaba-login-box")); driver.SwitchTo().Frame(iframe); // login if login form is present if (SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@id='fm-login-id']")) && SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@id='fm-login-password']"))) { IWebElement username = driver.FindElement(By.XPath("//input[@id='fm-login-id']")); IWebElement password = driver.FindElement(By.XPath("//input[@id='fm-login-password']")); username.Clear(); username.SendKeys(aliExpressUsername); password.Clear(); password.SendKeys(aliExpressPassword); // use password field to submit form password.Submit(); var waitLoginIFrame = new WebDriverWait(driver, TimeSpan.FromSeconds(30)); waitLoginIFrame.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete")); } // switch back to main frame driver.SwitchTo().DefaultContent(); } try { var waitMainPage = new WebDriverWait(driver, TimeSpan.FromSeconds(30)); waitMainPage.Until(SeleniumExtras.WaitHelpers.ExpectedConditions.UrlToBe("https://www.aliexpress.com/")); } catch (WebDriverTimeoutException) { Console.WriteLine("Timeout - Logged in to AliExpress to late. Stopping."); return(aliExpressOrders); } // go to order list driver.Navigate().GoToUrl("https://trade.aliexpress.com/orderList.htm"); // identify how many pages on order page (1/20) var tuple = GetAliExpressOrderPageNumber(driver); int curPage = tuple.Item1; int numPages = tuple.Item2; Console.WriteLine("Found {0} Pages", numPages); // scrape one and one page for (int i = 1; i <= numPages; i++) { // if this method returns false, it means we have reached the from date if (!ScrapeAliExpressOrderPage(aliExpressOrders, driver, i, from)) { break; } } driver.Close(); return(aliExpressOrders); }