/// <summary> /// Empties all product field TextBoxes on the main form, and clears the currentProduct variable /// </summary> private void ClearAllProductFields() { UpdateTextBox(txtbxStore, string.Empty); UpdateTextBox(txtbxItemNum, string.Empty); UpdateTextBox(txtbxBrandName, string.Empty); UpdateTextBox(txtbxProductName, string.Empty); UpdateTextBox(txtbxRegularPrice, string.Empty); UpdateTextBox(txtbxSalePrice, string.Empty); UpdateTextBox(txtbxUnitSize, string.Empty); // Clear lstbxPromotions { if (lstbxPromotions.InvokeRequired) { lstbxPromotions.Invoke(new MethodInvoker(() => lstbxPromotions.Items.Clear())); } else { lstbxPromotions.Items.Clear(); } } currentProduct = new Shipt_Product(); }
/// <summary> /// Whenever the browser lands on a product page, load the info onto the form and into currentProduct variable /// </summary> /// <param name="sender"></param> /// <param name="e">The object containing the product information</param> private void FormMain_LandedOnProductPageEvent(object sender, Shipt_Product e) { UpdateTextBox(txtbxStore, e.Store); UpdateTextBox(txtbxItemNum, e.Product_ID.ToString()); UpdateTextBox(txtbxBrandName, e.Brand_Name); UpdateTextBox(txtbxProductName, e.Product_Name); UpdateTextBox(txtbxRegularPrice, e.Regular_Price.ToString()); UpdateTextBox(txtbxSalePrice, e.Sale_Price.ToString()); UpdateTextBox(txtbxUnitSize, e.PricePerUnitFormatted()); // Clear lstbxPromotions and insert any promos if applicable { if (lstbxPromotions.InvokeRequired) { lstbxPromotions.Invoke(new MethodInvoker(() => lstbxPromotions.Items.Clear())); } else { lstbxPromotions.Items.Clear(); } if (e.Promotions.Count > 0) { if (lstbxPromotions.InvokeRequired) { lstbxPromotions.Invoke(new MethodInvoker(() => { foreach (var promo in e.Promotions) { lstbxPromotions.Items.Add(promo); } })); } else { foreach (var promo in e.Promotions) { lstbxPromotions.Items.Add(promo); } } } } if (btnTrackItem.InvokeRequired) { btnTrackItem.Invoke(new MethodInvoker(() => btnTrackItem.Enabled = true)); } else { btnTrackItem.Enabled = true; } currentProduct = e; }
/// <summary> /// The thread that controls the browser, and everything relevant to it /// </summary> private void t_Crawl() { try { browserRunning = true; ChromeOptions chromeOptions = new ChromeOptions(); ChromeDriverService driverService = ChromeDriverService.CreateDefaultService(); HtmlAgilityPack.HtmlDocument FullWebPage; //chromeOptions.AddArguments("headless"); chromeOptions.PageLoadStrategy = PageLoadStrategy.Normal; driverService.HideCommandPromptWindow = true; // Open Browser using (var browser = new ChromeDriver(driverService, chromeOptions)) { browser.Navigate().GoToUrl(URL_Login); // Wait for Url to change from URL_Login to URL_MainHomePage while (browser.Url != URL_MainHomePage) { if (!requestBrowserAbort) { Thread.Sleep(100); } else { return; } } // Let's check if we're logged in by looking for an address in the header of the page bool isLoggedIn = false; string hasAddressInHeader = browser.FindElements(By.XPath("//a[@class=\"pointer link darkness\"][@href=\"/account/addresses\"]//span")).First().Text; if (!string.IsNullOrEmpty(hasAddressInHeader)) { isLoggedIn = true; } if (isLoggedIn) { WebDriverWait wait = new WebDriverWait(browser, TimeSpan.FromSeconds(5)); wait.PollingInterval = TimeSpan.FromMilliseconds(50); // Get the addresses on the account, and the available stores at each address { wait.Until(ExpectedConditions.ElementExists(By.XPath("//button[@data-test=\"ShoppingStoreSelect-storeView\"]"))).Click(); // Trigger event to bring up the Select Store screen wait.Until(ExpectedConditions.ElementExists(By.XPath("//button[@id=\"SelectAddress-select\"][@data-test=\"Select-button\"]"))).Click(); // Trigger event to drop down list of addresses wait.Until(ExpectedConditions.ElementExists(By.XPath("//li[@data-test=\"Dropdown-option\"]"))); int numOfAddresses = browser.FindElements(By.XPath("//li[@data-test=\"Dropdown-option\"]")).Count; // Get count of addresses for (int i = 0; i < numOfAddresses; i++) { browser.FindElement(By.XPath("//button[@id=\"SelectAddress-select\"][@data-test=\"Select-button\"]")).Click(); if (i > 0) { wait.Until(ExpectedConditions.ElementExists(By.XPath("//li[@data-test=\"Dropdown-option\"]"))); browser.FindElements(By.XPath("//li[@data-test=\"Dropdown-option\"]"))[i].Click(); } Shipt_Available_Stores avail_stores = new Shipt_Available_Stores(); avail_stores.Delivery_Address = browser.FindElement(By.XPath("//div[@id=\"SelectAddress-select-selected-option-label\"]")).Text; wait.Until(ExpectedConditions.ElementExists(By.XPath("//div[@class=\"cf\"]"))); var Stores = browser.FindElements(By.XPath("//div[@class=\"cf\"]//div//div//p")); foreach (var store in Stores) { avail_stores.AddStore(store.Text); } // TODO: This event might be causing a huge delay in the program ReportAvailableStoresEvent?.Invoke(this, avail_stores); } browser.FindElement(By.XPath("//button[@id=\"SelectAddress-select\"][@data-test=\"Select-button\"]")).Click(); browser.FindElements(By.XPath("//li[@data-test=\"Dropdown-option\"]"))[0].Click(); browser.FindElement(By.XPath("//button[@data-test=\"Choose Store-modal-close\"]")).Click(); } string currURL = browser.Url; while (!requestBrowserAbort) { if (currURL != browser.Url) // Check if the URL has changed { // If the user navigated to a product page if (browser.Url.Contains(@"shop.shipt.com/products/")) { Shipt_Product product = new Shipt_Product(); /* * // I thought this was a good idea in case I change the type of Product_ID, I knew I'd forget to update this * // but it just creates more work than it's saving. * Type type = product.Product_ID.GetType(); * type.GetMethod("Parse").Invoke(null, new object[] { browser.Url.Substring(browser.Url.LastIndexOf("/")) } ); */ product.Address = browser.FindElements(By.XPath("//a[@class=\"pointer link darkness\"][@href=\"/account/addresses\"]//span")).First().Text; wait.Until(ExpectedConditions.ElementExists(By.XPath("//div[@data-test=\"ProductDetail-product-name\"]"))); // Product_ID product.Product_ID = uint.Parse(browser.Url.Substring(browser.Url.LastIndexOf("/") + 1)); // Store if (browser.FindElements(By.XPath("//span[@data-test=\"ShoppingStoreSelect-storeView-storeName\"]")).Count > 0) { product.Store = browser.FindElement(By.XPath("//span[@data-test=\"ShoppingStoreSelect-storeView-storeName\"]")).Text; } // Brand_Name if it exists if (browser.FindElements(By.XPath("//span[@data-test=\"ProductDetail-brand-name\"]")).Count > 0) { product.Brand_Name = browser.FindElement(By.XPath("//span[@data-test=\"ProductDetail-brand-name\"]")).Text; } // Product_Name if (browser.FindElements(By.XPath("//div[@data-test=\"ProductDetail-product-name\"]")).Count > 0) { product.Product_Name = browser.FindElement(By.XPath("//div[@data-test=\"ProductDetail-product-name\"]")).Text; } // Sale_Price and Regular_Price if product is on sale if (browser.FindElements(By.XPath("//div[@data-test=\"ProductDetail-product-sale-price\"]")).Count > 0) { /* * TIL the two FindElements inline are redundant * * string temp_Sale_Price = browser.FindElement(By.XPath("//div[@data-test=\"ProductDetail-product-sale-price\"]")).FindElement(By.XPath("//span[@class=\"mr1 title-2 tomato\"]")) * .Text * .Substring(1); * * string temp_Regular_Price = browser.FindElement(By.XPath("//div[@data-test=\"ProductDetail-product-sale-price\"]")).FindElement(By.XPath("//span[@class=\"mr1 body-2 gray strike\"]")) * .Text * .Substring(1); */ ReadOnlyCollection <IWebElement> Prices = browser.FindElements(By.XPath("//div[@data-test=\"ProductDetail-product-sale-price\"]//span")); if (Prices.Count >= 2) { string temp_Sale_Price = Prices[0].Text.Substring(1); string temp_Regular_Price = Prices[1].Text.Substring(1); try { product.Sale_Price = Convert.ToDecimal(temp_Sale_Price); product.Regular_Price = Convert.ToDecimal(temp_Regular_Price); } catch (Exception) { throw; } } else { //TODO: What happens if/when we don't have two elements for discount and regular price? } } // If product not on sale, it should have a regular price else if (browser.FindElements(By.XPath("//div[@data-test=\"ProductDetail-product-price\"]")).Count > 0) { string temp_Regular_Price = browser.FindElement(By.XPath("//div[@data-test=\"ProductDetail-product-price\"]")).Text.Substring(1); try { product.Regular_Price = Convert.ToDecimal(temp_Regular_Price); } catch (Exception) { throw; } } // This shouldn't be reached, but this should be handled else { //TODO: If product doesn't have a sale price or a regular price, what do we do? } // Subtext for Units and Unit_Type if (browser.FindElements(By.XPath("//div[@data-test=\"ProductDetail-subtext\"]")).Count > 0) { //TODO: Sometimes, subtext looks like "12 ct; 12 fl oz". Do we want to handle this a little differently? //TODO: "1/2 gal" won't parse to decimal. string temp_Subtext = browser.FindElement(By.XPath("//div[@data-test=\"ProductDetail-subtext\"]")).Text; if (char.IsDigit(temp_Subtext[0])) // TODO: Probably not a good idea to hard-code this { string temp_Units = temp_Subtext.Substring(0, (temp_Subtext.IndexOf(' '))).Replace("$", ""); string temp_UnitType = temp_Subtext.Substring(temp_Subtext.IndexOf(' ') + 1); try { product.Units = Convert.ToDecimal(temp_Units); } catch (Exception) { throw; } product.Unit_Type = temp_UnitType; } else { product.Units = 1; product.Unit_Type = temp_Subtext; } } // If product is BOGO if (browser.FindElements(By.XPath("//div[@data-test=\"ProductDetail-bogo-text\"]")).Count > 0) { product.AddPromotion("Buy 1, Get 1 Free"); } // If item has a promotion if (browser.FindElements(By.XPath("//button[@data-test=\"ProductDetail-feature-promotion\"]")).Count > 0) { string temp_Promotion = browser.FindElement(By.XPath("//button[@data-test=\"ProductDetail-feature-promotion\"]//div//div")).Text; product.AddPromotion(temp_Promotion); } LandedOnProductPageEvent?.Invoke(this, product); } // Check if the user left a product page for anything other than a product page else if (currURL.Contains(URL_Prefix_Product) && !browser.Url.Contains(URL_Prefix_Product)) { LeftProductPageEvent?.Invoke(this, new EventArgs()); } currURL = browser.Url; } } } } requestBrowserAbort = false; browserRunning = false; if (!mainFormIsClosing) { //Set btnCrawl text to "Start" if (btnCrawl.InvokeRequired) { btnCrawl.Invoke(new MethodInvoker(() => btnCrawl.Text = "Start Browser")); } else { btnCrawl.Text = "Start Browser"; } } } catch (ThreadAbortException) { requestBrowserAbort = false; browserRunning = false; if (!mainFormIsClosing) { //Set btnCrawl text to "Start" if (btnCrawl.InvokeRequired) { // This seems to lock the application and keep it alive, even though btnCrawl.IsDisposed and btnCrawl.Disposing are false btnCrawl.Invoke(new MethodInvoker(() => btnCrawl.Text = "Start Browser")); } else { btnCrawl.Text = "Start Browser"; } } } }
/// <summary> /// Add new product to the Products table.<para/> /// Query's the Addresses table for a match for use at Stores.Address_id and Products.Address_id.<para/> /// Query's the Stores table for a match for use at Products.Store_id.<para/> /// Query's the Products table for existing Products.Address_id, Products.Store_id, and Products.Product_id. /// </summary> /// <param name="product"></param> /// <returns></returns> public static bool InsertProduct(Shipt_Product product) { if (!DatabaseExists()) { CreateDatabase(); } using (SQLiteConnection connection = new SQLiteConnection(LoadConnectionString())) { int Address_ID = 0; SQLiteCommand QueryAddressID = new SQLiteCommand("SELECT Addresses.id FROM Addresses WHERE (Address LIKE ?)", connection); QueryAddressID.Parameters.AddWithValue("Address", product.Address + "%"); int Store_ID = 0; SQLiteCommand QueryStoreID = new SQLiteCommand("SELECT Stores.id FROM Stores WHERE (Address_id = ?) AND (Store = ?)", connection); SQLiteCommand QueryProductID = new SQLiteCommand("SELECT * FROM Products WHERE (Address_id = ?) AND (Store_id = ?) AND (Product_id = ?)", connection); QueryProductID.Parameters.AddWithValue("Product_id", product.Product_ID); SQLiteCommand InsertProduct = new SQLiteCommand("INSERT INTO Products (Address_id, Store_id, Product_id, Brand_Name, Product_Name, Units, Unit_Type, Active) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", connection); try { connection.Open(); SQLiteDataReader AddressReader = QueryAddressID.ExecuteReader(); // If AddressReader returned at least 1 row if (AddressReader.HasRows) { while (AddressReader.Read()) { Address_ID = Convert.ToInt32(AddressReader["id"]); if (AddressReader.Read()) { MessageBox.Show("While reading the Addresses table, it somehow returned multiple rows for one address. You should have this checked out by a professional!", "Addresses returned multiple rows", MessageBoxButtons.OK, MessageBoxIcon.Warning); } AddressReader.Close(); break; } QueryStoreID.Parameters.AddWithValue("Address_id", Address_ID); QueryStoreID.Parameters.AddWithValue("Store", product.Store); SQLiteDataReader StoreReader = QueryStoreID.ExecuteReader(); // If StoreReader returned at least 1 row if (StoreReader.HasRows) { while (StoreReader.Read()) { Store_ID = Convert.ToInt32(StoreReader["id"]); if (StoreReader.Read()) { MessageBox.Show("While reading the Stores table, it somehow returned multiple rows for one address and store. You should have this checked out by a professional!", "Stores returned multiple rows", MessageBoxButtons.OK, MessageBoxIcon.Warning); } StoreReader.Close(); break; } QueryProductID.Parameters.AddWithValue("Address_id", Address_ID); QueryProductID.Parameters.AddWithValue("Store_id", Store_ID); SQLiteDataReader ProductReader = QueryProductID.ExecuteReader(); // If ProductReader returned at least 1 row, then the product is already being tracked if (ProductReader.HasRows) { // TODO: Handle the case when a product already exists in the Products table ProductReader.Close(); } // The product does not yet exist else { InsertProduct.Parameters.AddWithValue("Address_id", Address_ID); InsertProduct.Parameters.AddWithValue("Store_id", Store_ID); InsertProduct.Parameters.AddWithValue("Product_id", product.Product_ID); InsertProduct.Parameters.AddWithValue("Brand_Name", product.Brand_Name); InsertProduct.Parameters.AddWithValue("Product_Name", product.Product_Name); InsertProduct.Parameters.AddWithValue("Units", product.Units); InsertProduct.Parameters.AddWithValue("Unit_Type", product.Unit_Type); InsertProduct.Parameters.AddWithValue("Active", "True"); InsertProduct.ExecuteNonQuery(); } } } return(true); } catch (Exception ex) { Console.WriteLine(ex.Message); return(false); } finally { connection.Close(); } } }