public async Task <IEnumerable <Review> > GetReview(string asin) { WebScrapper scrapper = new WebScrapper(_logger); List <Review> reviews = await scrapper.ScrapReview(asin); return(reviews.ToArray()); }
public async Task GetMatchingURLs_should_return_null_on_Bing_for_vjdotcom() { WebScrapper ws = new WebScrapper(); var results = await ws.GetMatchingURLs("test", Core.Enums.SearchEngine.Bing, new Uri("https://vj.com")); Assert.IsTrue(!results.Any()); }
public static SouqProductBucket GetProductBucket(string productFullId) { retry: SyncSouqCookies(); var url = $"https://egypt.souq.com/eg-en/{productFullId}/i/"; try { var document = WebScrapper.GetDownloadHtml(url); var globalBucketNode = document.DocumentNode.Descendants().First(l => l.Name == "script" && l.InnerHtml.Contains("globalBucket")); var globalBucketData = globalBucketNode.InnerText.Replace("var globalBucket =", "").TrimStart().TrimEnd(); return(JsonConvert.DeserializeObject <SouqProductBucket>(globalBucketData)); } catch (HtmlScapperException ex) { if (ex.InnerException.Message == "Too many automatic redirections were attempted.") { return(null); } goto retry; } }
public static SouqDeliveryInfo GetDeliveryInfo(string cityCode, string cityName, string unitId, string hitsCfs, string hitsCfsMeta) { retry: SyncSouqCookies(); var url = $"https://egypt.souq.com/eg-en/item_one.php?action=estimated_delivery¤tCityISO={cityCode}¤tCity={cityName}&" + $"idUnit={unitId}&hitsCfs={hitsCfs}&hitsCfsMeta={hitsCfsMeta.ToPercentageEncoding(1)}"; try { var obj = WebScrapper.GetDownloadJson <SouqDeliveryInfo>(url, headers: new Dictionary <string, string>() { { "X-Requested-With", "XMLHttpRequest" } }, cookies: Cookies); return(obj); } catch (HtmlScapperException ex) { if (ex.InnerException.Message.Contains("Too many automatic redirections were attempted")) { throw; } goto retry; } }
public async Task GetMatchingURLs_should_return_result_on_Bing_for_InfoTrack() { WebScrapper ws = new WebScrapper(); var results = await ws.GetMatchingURLs("test", Core.Enums.SearchEngine.Bing, new Uri("https://www.infotrack.com.au")); Assert.IsTrue(results.Any()); }
public static SouqCities GetCities() { retry: SyncSouqCookies(); var url = $"https://egypt.souq.com/eg-en/item_one.php"; try { var obj = WebScrapper.PostDownloadJson <SouqCities>(url, headers: new Dictionary <string, string>() { { "X-Requested-With", "XMLHttpRequest" } }, parameters: new Dictionary <string, string>() { { "action", "get_cities" }, { "country_code", "eg" }, { "selected_city", "CAI" } }); return(obj); } catch (HtmlScapperException ex) { goto retry; } }
private void MainForm_Load(object sender, EventArgs e) { this.ClientSize = Properties.Settings.Default.formSize; Visibility.Update(); progressMain.ProgressText = "Updating Name Lookups from abgx.net"; _scanner = new GameScanner(); _scanner.OnGameProcessed += _loader_OnGameScanned; _scanner.OnStatusUpdate += _loader_OnStatusUpdate; _scanner.OnFinish += _loader_OnScanFinish; _scrapper = new WebScrapper(); _scrapper.OnGameProcessed += _OnGameProcessed; _scrapper.OnStatusUpdate += _loader_OnStatusUpdate; _scrapper.OnFinish += _loader_OnFinish; _abgx = new AbgxChecker(); _abgx.OnFinish += _loader_OnFinish; _abgx.OnGameProcessed += _OnGameProcessed; _abgx.OnStatusUpdate += _loader_OnStatusUpdate; _move = new MoveProcessor(); _move.OnFinish += _loader_OnFinish; _move.OnGameProcessed += _OnGameProcessed; _move.OnStatusUpdate += _loader_OnStatusUpdate; _info = new ScannerInfo(); progressMain.ProgressText = ""; progressSub.ProgressText = ""; if (Program.Wizard) { menuSettings_Click(null, null); } }
public eAlertProcess() { _serviceName = "eAlert Resender"; scrapper = new WebScrapper(); scrapper.baseUrl = ConfigurationManager.AppSettings["TeleCAREWebsite"]; five9URL = ConfigurationManager.AppSettings["Five9URL"]; SendEmailTo = ConfigurationManager.AppSettings["SendMailTo"]; }
public static IEnumerable <SouqSellerReview> GetSellerReview(string sellerId, int page = 0, int pageLimit = -1) { List <SouqSellerReview> tmpList = new List <SouqSellerReview>(); int tmpPage = 0; bool stillAvailable; do { retry: SyncSouqCookies(); var url = $"https://egypt.souq.com/eg-en/{sellerId.ToPercentageEncoding(SouqApiConstants.PercentageEncodingLevel)}/p/profile.html"; try { var obj = WebScrapper.PostDownloadJson <SouqSellerReview>(url, headers: new Dictionary <string, string>() { { "X-Requested-With", "XMLHttpRequest" } }, parameters: new Dictionary <string, string>() { { "action", "show_more" }, { "source", "seller_profile" }, { "row", $"{tmpPage}" }, { "seller", $"{sellerId}" }, { "filter", "" }, { "r_url", $"{url}" } }); tmpList.Add(obj); stillAvailable = obj.bShowMore; if (pageLimit < 0 || page < page + pageLimit) { tmpPage++; } else { stillAvailable = false; } } catch (HtmlScapperException ex) { /*Console Write*/ goto retry; } } while (stillAvailable); return(tmpList); }
public static HtmlDocument GetSellerProfile(string sellerId) { retry: var url = $"https://egypt.souq.com/eg-en/{sellerId.ToPercentageEncoding(SouqApiConstants.PercentageEncodingLevel)}/p/profile.html"; SyncSouqCookies(); try { var document = WebScrapper.GetDownloadHtml(url); return(document); } catch (HtmlScapperException ex) { goto retry; } }
public static HtmlDocument GetCategories() { retry: var url = $"https://egypt.souq.com/eg-en/shop-all-categories/c/?ref=nav"; SyncSouqCookies(); try { var document = WebScrapper.GetDownloadHtml(url); return(document); } catch (HtmlScapperException ex) { goto retry; } }
public static IEnumerable <SouqProductReview> GetProductReview(string productId, int page = 0, int pageLimit = -1) { List <SouqProductReview> tmpList = new List <SouqProductReview>(); int tmpPage = page; bool stillAvailable; do { retry: SyncSouqCookies(); var url = $"https://egypt.souq.com/eg-en/reviews.php?page={tmpPage + 1}&action=get_reviews&id_item={productId}"; try { var obj = WebScrapper.GetDownloadJson <SouqProductReview>(url, headers: new Dictionary <string, string>() { { "X-Requested-With", "XMLHttpRequest" } }); tmpList.Add(obj); stillAvailable = obj.show_more; if (pageLimit < 0 || page < page + pageLimit) { tmpPage++; } else { stillAvailable = false; } } catch (HtmlScapperException ex) { /*Console Write*/ goto retry; } } while (stillAvailable); return(tmpList); }
public void Run() { var credentialSection = Program.configuration.GetSection("Credentials"); var credentials = credentialSection.GetChildren().ToArray(); // Extraction and Transform if (m_optionsCurrent != null) { scrapper = new WebScrapper(m_optionsCurrent.Flight, credentials[0].Value, credentials[1].Value, "https://www.flightradar24.com/21.33,-94.49/6", false); } else if (m_optionsHistorical != null) { scrapper = new WebScrapper(m_optionsHistorical.Flight, credentials[0].Value, credentials[1].Value, "https://www.flightradar24.com/21.33,-94.49/6", true); } if (scrapper.Run()) { LoadDataIntoDatabase(); } }
/// <summary> /// Creates a web scrapper for provided <paramref name="url"/> and retrieves website's info. /// </summary> public static async Task <IApiResponse> GetWebsiteInfo(string url) { WebScrapper webScrapper; try { webScrapper = await WebScrapper.Create(url); } catch (BadUrlException) { return(new ErrorResponse { Error = "Bad url." }); } var finalUrl = webScrapper.GetWebsiteUrl(); var title = webScrapper.GetWebsiteTitle(); var response = new UrlSolverResponse { Url = finalUrl, Title = title, }; var notes = new List <string>(); if (finalUrl == url) { notes.Add("There was no redirection or redirection is made outside HTTP header."); } if (title is null) { notes.Add("Title is probably dynamically set after the page loads."); } if (notes.Count > 0) { response.Notes = notes; } return(response); }
private static void SyncSouqCookies() { if (ExpireSession > DateTime.Now) { return; } var url = "https://egypt.souq.com/eg-en/shop-all-categories/c/?ref=nav"; retry: try { var obj = (WebScrapper.GetHeaders(url))["Set-Cookie"]; var cc = GetAllCookiesFromHeader(obj, url); ExpireSession = DateTime.Now.AddYears(1).AddDays(-2); Cookies = cc; cookieString = obj; } catch (Exception ex) { goto retry; } }
public PhysicianStatusProcessor() { _serviceName = "Physician Status Mover Service"; scrapper = new WebScrapper(); scrapper.baseUrl = ConfigurationManager.AppSettings["TeleCAREWebsite"]; }
public void NaiveMouser_Test3() { Assert.AreEqual("Thick Film Resistors - SMD 2.2K OHM 1% Learn More", WebScrapper.NaiveMouser("RC0603FR-072K2L")); }
public void NaiveMouser_Test2() { Assert.AreEqual("Ferrite Beads 220 OHM HIGH SPEED SIGNAL Learn More", WebScrapper.NaiveMouser("BLM15BD221SN1D")); }
public void NaiveMouser_Test() { Assert.AreEqual("Multilayer Ceramic Capacitors MLCC - SMD/SMT 0402 4.7uF 6.3volts *Derate Voltage/Temp Learn More", WebScrapper.NaiveMouser("GRM155R60J475ME87D")); }
public void NaiveElement14_Test2() { Assert.AreEqual("MURATA - Ferrite Bead, 0402 [1005 Metric], 220 ohm, 300 mA, BLM15B Series, 0.4 ohm, ± 25%", WebScrapper.NaiveElement14("BLM15BD221SN1D")); }
public void NaiveElement14_Test() { Assert.AreEqual("YAGEO - SMD Chip Resistor, 2.2 kohm, RC Series, 75 V, Thick Film, 0603 [1608 Metric], 100 mW ", WebScrapper.NaiveElement14("RC0603FR-072K2L")); }
public void NaiveRS_Test3() { Assert.AreEqual("Murata 10nF Multilayer Ceramic Capacitor MLCC 6.3V dc ±10% X7R Dielectric 0402 (1005M) SMD, Max. Temp. +125°C ", WebScrapper.NaiveRS("GRM155R70J103KA01D")); }
public void NaiveRS_Test2() { Assert.AreEqual("Murata Ferrite Bead (Chip Ferrite Bead), 1 x 0.5 x 0.5mm (0402 (1005M)), 220Ω impedance at 100 MHz ", WebScrapper.NaiveRS("BLM15BD221SN1D")); }
public void NaiveRS_Test() { Assert.AreEqual("TE Connectivity CR2032 PCB Battery Holder, Leaf Spring Contact ", WebScrapper.NaiveRS("796136-1")); }
public void NaiveDigikey_Test2() { Assert.AreEqual("DIODE ARRAY SCHOTTKY 30V SOT23 ", WebScrapper.NaiveDigikey("BAT54S, 215")); }
public void NaiveDigikey_Test() { Assert.AreEqual("CAP CER 1000PF 100V C0G 0603 ", WebScrapper.NaiveDigikey("C1608C0G2A102J080AA")); }
public void Element14API_GetDescriptionTest() { Assert.AreEqual("ABRACON - ABM3B-12.000MHZ-B2-T - CRYSTAL, 12MHZ, 18PF, 5 X 3.2MM", WebScrapper.Element14API_GetDescription("ABM3B-12.000MHZ-B2-T")); }
public void NaiveMouser_Test4() { Assert.AreEqual("Crystals +/-20ppm 12MHZ", WebScrapper.NaiveMouser("ABM3B-12.000MHZ-B2-T")); }
public void API_MouserTest() { Assert.AreEqual("Crystals +/-20ppm 12MHZ", WebScrapper.MouserAPI_GetDescription("ABM3B-12.000MHZ-B2-T")); }
public async Task CallURL_should_fire_exception_for_invalid_url() { var result = await WebScrapper.CallUrl("https://www.vj.au"); }