// Threaded function
private void scrapeBook(string ISBN)
{
this.Invoke((MethodInvoker) delegate()
{
updateLabel.Text = ISBN;
detailsListBox.Text += "Scraping " + ISBN + "... ";
});
AmazonScraper amazon = new AmazonScraper();
try
{
string url = "https://www.amazon.com/Clash-Kings-Song-Fire-Book/dp/" + ISBN + "/ref=tmm_hrd_swatch_0?_encoding=UTF8&qid=1559015792&sr=8-2";
string page = amazon.getPage(url);
Book book = amazon.getBook(ISBN, page);
pushBookToTable(book);
}
catch (Exception ex)
{
this.Invoke((MethodInvoker) delegate()
{
detailsListBox.Text += "error: " + ex.Message + "!";
errorListText.Text += ISBN + Environment.NewLine;
});
}
this.Invoke((MethodInvoker) delegate()
{
progress.PerformStep();
detailsListBox.Text += Environment.NewLine;
});
}
protected static List <IScraper> InitScrapers(Dictionary <int, string> urlDic)
{
List <IScraper> scraperList = new List <IScraper>();
foreach (KeyValuePair <int, string> item in urlDic)
{
IScraper scraper = null;
if (Utility.findString(item.Value, "amazon"))
{
scraper = new AmazonScraper(item.Key, item.Value);
}
else if (Utility.findString(item.Value, "backmarket"))
{
scraper = new BackMarketScraper(item.Key, item.Value);
}
if (scraper != null)
{
scraperList.Add(scraper);
}
}
if (scraperList.Count == 0)
{
throw new System.InvalidOperationException("There is no scraper recognized in the list!");
}
return(scraperList);
}
private async Task AmazonScrape(string term)
{
var amazonnsearchurl = AmazonBaseUrl + term;
AmazonScraper amazonScraper = new AmazonScraper();
_jsonContainer = await amazonScraper.BeginScrapeAsync(amazonnsearchurl);
}
public static Thread CreateThread(Uri url)
{
// determine the domain
Scraper scraper = null;
Thread thread = null;
switch (url.Host)
{
case "www.bhphotovideo.com":
{
scraper = new BHScraper(url);
break;
}
case "www.newegg.com":
{
scraper = new NewEggScraper(url);
break;
}
case "www.amazon.com":
{
scraper = new AmazonScraper(url);
break;
}
case "www.bestbuy.com":
{
scraper = new BestBuyScraper(url);
break;
}
case "www.microcenter.com":
{
scraper = new MicroCenterScraper(url);
break;
}
case "www.amd.com":
{
scraper = new AmdScraper(url);
break;
}
default: throw new ArgumentException("Could not find scraper for url: " + url.Host);
}
try
{
thread = new Thread(scraper.Scrape);
}
catch (Exception e)
{
Console.WriteLine(e);
}
return(thread);
}
public async Task <IActionResult> Result(string search)
{
var amazon = new AmazonScraper();
var ebay = new EbayScraper();
var jLewis = new JohnLewisScraper();
Hashtable amazonResults;
Hashtable ebayResults;
Hashtable jLewisResults;
ViewBag.ProductDescription = search;
// scrape retailers
try
{
jLewisResults = await jLewis.ScrapePricesForProduct(search);
ebayResults = await ebay.ScrapePricesForProduct(search);
amazonResults = await amazon.ScrapePricesForProduct(search);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
// create new result
var result = new Result()
{
Date = DateTime.Now,
AmazonPrice = Convert.ToDouble(amazonResults["Formatted Price"]),
EbayPrice = Convert.ToDouble(ebayResults["Formatted Price"]),
JohnLewisPrice = Convert.ToDouble(jLewisResults["Formatted Price"]),
AmazonLink = amazonResults["Product Link"].ToString(),
AmazonHeading = amazonResults["Product Heading"].ToString(),
EbayLink = ebayResults["Product Link"].ToString(),
EbayHeading = ebayResults["Product Heading"].ToString(),
JohnLewisHeading = jLewisResults["Product Heading"].ToString(),
JohnLewisLink = jLewisResults["Product Link"].ToString(),
};
//create Saved Search
var savedSearch = new SavedSearch()
{
CreatedDate = DateTime.Now,
Description = search,
};
// create save search view model
var viewModel = new SaveSearchViewModel()
{
Result = result,
SavedSearch = savedSearch
};
return(View(viewModel));
}
public ActionResult Index(string searchTerm)
{
var viewModel = new IndexViewModel();
viewModel.SearchTerm = searchTerm;
if (!String.IsNullOrEmpty(searchTerm))
{
viewModel.Items = AmazonScraper.Search(searchTerm);
}
return(View(viewModel));
}
public static async Task UpdateSearches()
{
Console.WriteLine("task started successfully");
using (var _context = new PriceTrackerContext())
{
var amazon = new AmazonScraper();
var ebay = new EbayScraper();
var jLewis = new JohnLewisScraper();
//get the saved searches
var savedSearches = _context.SavedSearch.OrderByDescending(x => x.CreatedDate).Include(x => x.Results).ToList();
foreach (var search in savedSearches)
{
Hashtable amazonResults;
Hashtable ebayResults;
Hashtable jLewisResults;
try
{
jLewisResults = await jLewis.ScrapePricesForProduct(search.Description);
ebayResults = await ebay.ScrapePricesForProduct(search.Description);
amazonResults = await amazon.ScrapePricesForProduct(search.Description);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
var result = new Result()
{
Date = DateTime.Now,
AmazonPrice = Convert.ToDouble(amazonResults["Formatted Price"]),
EbayPrice = Convert.ToDouble(ebayResults["Formatted Price"]),
JohnLewisPrice = Convert.ToDouble(jLewisResults["Formatted Price"]),
AmazonLink = amazonResults["Product Link"].ToString(),
AmazonHeading = amazonResults["Product Heading"].ToString(),
EbayLink = ebayResults["Product Link"].ToString(),
EbayHeading = ebayResults["Product Heading"].ToString(),
JohnLewisHeading = jLewisResults["Product Heading"].ToString(),
JohnLewisLink = jLewisResults["Product Link"].ToString(),
};
search.Results.Add(result);
await _context.SaveChangesAsync();
}
}
}
// When the search button is clicked
private void searchButton_Click(object sender, EventArgs e)
{
if (textBox1.Text.Length != 10)
{
return;
}
string url = "https://www.amazon.com/Clash-Kings-Song-Fire-Book/dp/" + textBox1.Text + "/ref=tmm_hrd_swatch_0?_encoding=UTF8&qid=1559015792&sr=8-2";
AmazonScraper scraper = new AmazonScraper();
// Restart progress bar and update label
{
progress.Value = 0;
updateLabel.Text = "Downloading webpage...";
Update();
}
// Retrieve the page
string page = scraper.getPage(url);
if (page == null)
{
showError("ISBN not valid!");
updateLabel.Text = "Click search to scrape for an ISBN";
progress.Value = 0;
return;
}
// Update progress bar and update label
{
progress.Value = 50;
updateLabel.Text = "Finding information...";
Update();
}
Book book = scraper.getBook(textBox1.Text, page);
progress.Value = 100;
if (book == null)
{
showError("ISBN not valid!");
return;
}
pushBookToTable(book);
updateLabel.Text = "Click search to scrape for an ISBN";
}
public ActionResult Index(string searchTerm)
{
var vm = new HomePageViewModel();
if (String.IsNullOrEmpty(searchTerm))
{
return(View(vm));
}
vm.SearchTerm = searchTerm;
var scraper = new AmazonScraper();
vm.Items = scraper.Scrape(searchTerm);
return(View(vm));
}
public static List <IScraper> InitScrapers(List <string> urlList)
{
List <IScraper> scraperList = new List <IScraper>();
foreach (string strUrl in urlList)
{
IScraper scraper = null;
if (Utility.findString(strUrl, "amazon"))
{
scraper = new AmazonScraper();
}
else if (Utility.findString(strUrl, "backmarket"))
{
scraper = new BackMarketScraper();
}
if (scraper != null)
{
scraperList.Add(scraper);
}
}
return(scraperList);
}
public List <AmazonResult> Search(string term)
{
return(AmazonScraper.Search(term));
}
public async Task UpdateResults()
{
Console.WriteLine($"Update Task started at {DateTime.Now}");
using (var _context = new PriceTrackerContext())
{
var amazon = new AmazonScraper();
var ebay = new EbayScraper();
var jLewis = new JohnLewisScraper();
//get the saved searches
var savedSearches = _context.SavedSearch.OrderByDescending(x => x.CreatedDate).Include(x => x.Results).ToList();
if (savedSearches.Count != 0)
{
foreach (var search in savedSearches)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($"scraping prices for search: {search.Description} , id {search.SavedSearchId}, created on: {search.CreatedDate}");
Console.ForegroundColor = ConsoleColor.Gray;
Hashtable amazonResults;
Hashtable ebayResults;
Hashtable jLewisResults;
try
{
jLewisResults = await jLewis.ScrapePricesForProduct(search.Description);
ebayResults = await ebay.ScrapePricesForProduct(search.Description);
amazonResults = await amazon.ScrapePricesForProduct(search.Description);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
var result = new Result()
{
Date = DateTime.Now,
AmazonPrice = Convert.ToDouble(amazonResults["Formatted Price"]),
EbayPrice = Convert.ToDouble(ebayResults["Formatted Price"]),
JohnLewisPrice = Convert.ToDouble(jLewisResults["Formatted Price"]),
AmazonLink = amazonResults["Product Link"].ToString(),
AmazonHeading = amazonResults["Product Heading"].ToString(),
EbayLink = ebayResults["Product Link"].ToString(),
EbayHeading = ebayResults["Product Heading"].ToString(),
JohnLewisHeading = jLewisResults["Product Heading"].ToString(),
JohnLewisLink = jLewisResults["Product Link"].ToString(),
};
//get the latest result for this saved search
var currentSearchResult = search.Results.OrderByDescending(x => x.Date).FirstOrDefault();
search.Results.Add(result);
await _context.SaveChangesAsync();
// compare the prices here
var emailBody = ComparePrices(result, currentSearchResult);
if (!string.IsNullOrEmpty(emailBody))
{
Console.WriteLine(SendNotificationEmail(emailBody));
}
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("completed ");
Console.ForegroundColor = ConsoleColor.Gray;
}
}
else
{
Console.WriteLine("No Saved Searches to update");
}
}
Console.WriteLine($"Update Task completed successfully at {DateTime.Now}");
}
public BackgroundEngine(InMemory inMemory, AmazonScraper amazonScraper)
{
_process = Task.Run(async() => await Consummer());
_inMemory = inMemory;
_amazonScraper = amazonScraper;
}
