/// <summary>
/// Whenever we browse the addresses and the stores available at that address, insert the address and the stores in the DB
/// </summary>
/// <param name="sender"></param>
/// <param name="e">The object containing the address and available stores</param>
private void FormMain_ReportAvailableStoresEvent(object sender, Shipt_Available_Stores e)
{
if (e.StoreNames.Count > 0)
{
DB_Manager.InsertAddress(e.Delivery_Address);
DB_Manager.InsertStores(e);
}
}
/// <summary>
/// The thread that controls the browser, and everything relevant to it
/// </summary>
private void t_Crawl()
{
try
{
browserRunning = true;
ChromeOptions chromeOptions = new ChromeOptions();
ChromeDriverService driverService = ChromeDriverService.CreateDefaultService();
HtmlAgilityPack.HtmlDocument FullWebPage;
//chromeOptions.AddArguments("headless");
chromeOptions.PageLoadStrategy = PageLoadStrategy.Normal;
driverService.HideCommandPromptWindow = true;
// Open Browser
using (var browser = new ChromeDriver(driverService, chromeOptions))
{
browser.Navigate().GoToUrl(URL_Login);
// Wait for Url to change from URL_Login to URL_MainHomePage
while (browser.Url != URL_MainHomePage)
{
if (!requestBrowserAbort)
{
Thread.Sleep(100);
}
else
{
return;
}
}
// Let's check if we're logged in by looking for an address in the header of the page
bool isLoggedIn = false;
string hasAddressInHeader = browser.FindElements(By.XPath("//a[@class=\"pointer link darkness\"][@href=\"/account/addresses\"]//span")).First().Text;
if (!string.IsNullOrEmpty(hasAddressInHeader))
{
isLoggedIn = true;
}
if (isLoggedIn)
{
WebDriverWait wait = new WebDriverWait(browser, TimeSpan.FromSeconds(5));
wait.PollingInterval = TimeSpan.FromMilliseconds(50);
// Get the addresses on the account, and the available stores at each address
{
wait.Until(ExpectedConditions.ElementExists(By.XPath("//button[@data-test=\"ShoppingStoreSelect-storeView\"]"))).Click(); // Trigger event to bring up the Select Store screen
wait.Until(ExpectedConditions.ElementExists(By.XPath("//button[@id=\"SelectAddress-select\"][@data-test=\"Select-button\"]"))).Click(); // Trigger event to drop down list of addresses
wait.Until(ExpectedConditions.ElementExists(By.XPath("//li[@data-test=\"Dropdown-option\"]")));
int numOfAddresses = browser.FindElements(By.XPath("//li[@data-test=\"Dropdown-option\"]")).Count; // Get count of addresses
for (int i = 0; i < numOfAddresses; i++)
{
browser.FindElement(By.XPath("//button[@id=\"SelectAddress-select\"][@data-test=\"Select-button\"]")).Click();
if (i > 0)
{
wait.Until(ExpectedConditions.ElementExists(By.XPath("//li[@data-test=\"Dropdown-option\"]")));
browser.FindElements(By.XPath("//li[@data-test=\"Dropdown-option\"]"))[i].Click();
}
Shipt_Available_Stores avail_stores = new Shipt_Available_Stores();
avail_stores.Delivery_Address = browser.FindElement(By.XPath("//div[@id=\"SelectAddress-select-selected-option-label\"]")).Text;
wait.Until(ExpectedConditions.ElementExists(By.XPath("//div[@class=\"cf\"]")));
var Stores = browser.FindElements(By.XPath("//div[@class=\"cf\"]//div//div//p"));
foreach (var store in Stores)
{
avail_stores.AddStore(store.Text);
}
// TODO: This event might be causing a huge delay in the program
ReportAvailableStoresEvent?.Invoke(this, avail_stores);
}
browser.FindElement(By.XPath("//button[@id=\"SelectAddress-select\"][@data-test=\"Select-button\"]")).Click();
browser.FindElements(By.XPath("//li[@data-test=\"Dropdown-option\"]"))[0].Click();
browser.FindElement(By.XPath("//button[@data-test=\"Choose Store-modal-close\"]")).Click();
}
string currURL = browser.Url;
while (!requestBrowserAbort)
{
if (currURL != browser.Url) // Check if the URL has changed
{
// If the user navigated to a product page
if (browser.Url.Contains(@"shop.shipt.com/products/"))
{
Shipt_Product product = new Shipt_Product();
/*
* // I thought this was a good idea in case I change the type of Product_ID, I knew I'd forget to update this
* // but it just creates more work than it's saving.
* Type type = product.Product_ID.GetType();
* type.GetMethod("Parse").Invoke(null, new object[] { browser.Url.Substring(browser.Url.LastIndexOf("/")) } );
*/
product.Address = browser.FindElements(By.XPath("//a[@class=\"pointer link darkness\"][@href=\"/account/addresses\"]//span")).First().Text;
wait.Until(ExpectedConditions.ElementExists(By.XPath("//div[@data-test=\"ProductDetail-product-name\"]")));
// Product_ID
product.Product_ID = uint.Parse(browser.Url.Substring(browser.Url.LastIndexOf("/") + 1));
// Store
if (browser.FindElements(By.XPath("//span[@data-test=\"ShoppingStoreSelect-storeView-storeName\"]")).Count > 0)
{
product.Store = browser.FindElement(By.XPath("//span[@data-test=\"ShoppingStoreSelect-storeView-storeName\"]")).Text;
}
// Brand_Name if it exists
if (browser.FindElements(By.XPath("//span[@data-test=\"ProductDetail-brand-name\"]")).Count > 0)
{
product.Brand_Name = browser.FindElement(By.XPath("//span[@data-test=\"ProductDetail-brand-name\"]")).Text;
}
// Product_Name
if (browser.FindElements(By.XPath("//div[@data-test=\"ProductDetail-product-name\"]")).Count > 0)
{
product.Product_Name = browser.FindElement(By.XPath("//div[@data-test=\"ProductDetail-product-name\"]")).Text;
}
// Sale_Price and Regular_Price if product is on sale
if (browser.FindElements(By.XPath("//div[@data-test=\"ProductDetail-product-sale-price\"]")).Count > 0)
{
/*
* TIL the two FindElements inline are redundant
*
* string temp_Sale_Price = browser.FindElement(By.XPath("//div[@data-test=\"ProductDetail-product-sale-price\"]")).FindElement(By.XPath("//span[@class=\"mr1 title-2 tomato\"]"))
* .Text
* .Substring(1);
*
* string temp_Regular_Price = browser.FindElement(By.XPath("//div[@data-test=\"ProductDetail-product-sale-price\"]")).FindElement(By.XPath("//span[@class=\"mr1 body-2 gray strike\"]"))
* .Text
* .Substring(1);
*/
ReadOnlyCollection <IWebElement> Prices = browser.FindElements(By.XPath("//div[@data-test=\"ProductDetail-product-sale-price\"]//span"));
if (Prices.Count >= 2)
{
string temp_Sale_Price = Prices[0].Text.Substring(1);
string temp_Regular_Price = Prices[1].Text.Substring(1);
try
{
product.Sale_Price = Convert.ToDecimal(temp_Sale_Price);
product.Regular_Price = Convert.ToDecimal(temp_Regular_Price);
}
catch (Exception)
{
throw;
}
}
else
{
//TODO: What happens if/when we don't have two elements for discount and regular price?
}
}
// If product not on sale, it should have a regular price
else if (browser.FindElements(By.XPath("//div[@data-test=\"ProductDetail-product-price\"]")).Count > 0)
{
string temp_Regular_Price = browser.FindElement(By.XPath("//div[@data-test=\"ProductDetail-product-price\"]")).Text.Substring(1);
try
{
product.Regular_Price = Convert.ToDecimal(temp_Regular_Price);
}
catch (Exception)
{
throw;
}
}
// This shouldn't be reached, but this should be handled
else
{
//TODO: If product doesn't have a sale price or a regular price, what do we do?
}
// Subtext for Units and Unit_Type
if (browser.FindElements(By.XPath("//div[@data-test=\"ProductDetail-subtext\"]")).Count > 0)
{
//TODO: Sometimes, subtext looks like "12 ct; 12 fl oz". Do we want to handle this a little differently?
//TODO: "1/2 gal" won't parse to decimal.
string temp_Subtext = browser.FindElement(By.XPath("//div[@data-test=\"ProductDetail-subtext\"]")).Text;
if (char.IsDigit(temp_Subtext[0])) // TODO: Probably not a good idea to hard-code this
{
string temp_Units = temp_Subtext.Substring(0, (temp_Subtext.IndexOf(' '))).Replace("$", "");
string temp_UnitType = temp_Subtext.Substring(temp_Subtext.IndexOf(' ') + 1);
try
{
product.Units = Convert.ToDecimal(temp_Units);
}
catch (Exception)
{
throw;
}
product.Unit_Type = temp_UnitType;
}
else
{
product.Units = 1;
product.Unit_Type = temp_Subtext;
}
}
// If product is BOGO
if (browser.FindElements(By.XPath("//div[@data-test=\"ProductDetail-bogo-text\"]")).Count > 0)
{
product.AddPromotion("Buy 1, Get 1 Free");
}
// If item has a promotion
if (browser.FindElements(By.XPath("//button[@data-test=\"ProductDetail-feature-promotion\"]")).Count > 0)
{
string temp_Promotion = browser.FindElement(By.XPath("//button[@data-test=\"ProductDetail-feature-promotion\"]//div//div")).Text;
product.AddPromotion(temp_Promotion);
}
LandedOnProductPageEvent?.Invoke(this, product);
}
// Check if the user left a product page for anything other than a product page
else if (currURL.Contains(URL_Prefix_Product) &&
!browser.Url.Contains(URL_Prefix_Product))
{
LeftProductPageEvent?.Invoke(this, new EventArgs());
}
currURL = browser.Url;
}
}
}
}
requestBrowserAbort = false;
browserRunning = false;
if (!mainFormIsClosing)
{
//Set btnCrawl text to "Start"
if (btnCrawl.InvokeRequired)
{
btnCrawl.Invoke(new MethodInvoker(() => btnCrawl.Text = "Start Browser"));
}
else
{
btnCrawl.Text = "Start Browser";
}
}
}
catch (ThreadAbortException)
{
requestBrowserAbort = false;
browserRunning = false;
if (!mainFormIsClosing)
{
//Set btnCrawl text to "Start"
if (btnCrawl.InvokeRequired)
{
// This seems to lock the application and keep it alive, even though btnCrawl.IsDisposed and btnCrawl.Disposing are false
btnCrawl.Invoke(new MethodInvoker(() => btnCrawl.Text = "Start Browser"));
}
else
{
btnCrawl.Text = "Start Browser";
}
}
}
}
/// <summary>
/// Add new stores to the Stores table.<para/>
/// Query's the Addresses table for a match for use at Stores.Address_id.<para/>
/// Query's the Stores table for existing Stores.id and Stores.Address_id.
/// </summary>
/// <param name="stores">The address and stores to be added.</param>
/// <returns>True if successfully added.</returns>
public static bool InsertStores(Shipt_Available_Stores stores)
{
if (!DatabaseExists())
{
CreateDatabase();
}
using (SQLiteConnection connection = new SQLiteConnection(LoadConnectionString()))
{
int Address_ID;
SQLiteCommand QueryStores = new SQLiteCommand("SELECT Stores.Store FROM Stores WHERE (Address_id = ?)", connection);
SQLiteCommand command = new SQLiteCommand("INSERT INTO Stores (Address_id, Store, Active) VALUES (?, ?, ?)", connection);
command.Parameters.Add("Address_id", DbType.Int32);
command.Parameters.Add("Store", DbType.String);
command.Parameters.Add("Active", DbType.String);
try
{
connection.Open();
Address_ID = FindAddress_id(connection, stores.Delivery_Address);
if (Address_ID > 0)
{
command.Parameters["Address_id"].Value = Address_ID;
List <string> modifiedStores = stores.StoreNames;
QueryStores.Parameters.AddWithValue("Address_id", Address_ID);
SQLiteDataReader reader = QueryStores.ExecuteReader();
// Take out any existing stores from modifiedStores so we don't spend time adding rows that already exist
while (reader.Read())
{
if (modifiedStores.Contains(reader[0].ToString()))
{
modifiedStores.Remove(reader[0].ToString());
}
}
foreach (string store in modifiedStores)
{
command.Parameters["Store"].Value = store;
command.Parameters["Active"].Value = "True";
command.ExecuteNonQuery();
}
}
return(true);
}
catch (Exception ex)
{
return(false);
}
finally
{
connection.Close();
command.Dispose();
}
}
}
