public async Task <IEnumerable <Review> > GetReview(string asin)
{
WebScrapper scrapper = new WebScrapper(_logger);
List <Review> reviews = await scrapper.ScrapReview(asin);
return(reviews.ToArray());
}
public async Task GetMatchingURLs_should_return_null_on_Bing_for_vjdotcom()
{
WebScrapper ws = new WebScrapper();
var results = await ws.GetMatchingURLs("test", Core.Enums.SearchEngine.Bing, new Uri("https://vj.com"));
Assert.IsTrue(!results.Any());
}
public static SouqProductBucket GetProductBucket(string productFullId)
{
retry:
SyncSouqCookies();
var url = $"https://egypt.souq.com/eg-en/{productFullId}/i/";
try
{
var document = WebScrapper.GetDownloadHtml(url);
var globalBucketNode = document.DocumentNode.Descendants().First(l => l.Name == "script" && l.InnerHtml.Contains("globalBucket"));
var globalBucketData =
globalBucketNode.InnerText.Replace("var globalBucket =", "").TrimStart().TrimEnd();
return(JsonConvert.DeserializeObject <SouqProductBucket>(globalBucketData));
}
catch (HtmlScapperException ex)
{
if (ex.InnerException.Message == "Too many automatic redirections were attempted.")
{
return(null);
}
goto retry;
}
}
public static SouqDeliveryInfo GetDeliveryInfo(string cityCode, string cityName, string unitId, string hitsCfs, string hitsCfsMeta)
{
retry:
SyncSouqCookies();
var url =
$"https://egypt.souq.com/eg-en/item_one.php?action=estimated_delivery¤tCityISO={cityCode}¤tCity={cityName}&" +
$"idUnit={unitId}&hitsCfs={hitsCfs}&hitsCfsMeta={hitsCfsMeta.ToPercentageEncoding(1)}";
try
{
var obj = WebScrapper.GetDownloadJson <SouqDeliveryInfo>(url, headers: new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
}, cookies: Cookies);
return(obj);
}
catch (HtmlScapperException ex)
{
if (ex.InnerException.Message.Contains("Too many automatic redirections were attempted"))
{
throw;
}
goto retry;
}
}
public async Task GetMatchingURLs_should_return_result_on_Bing_for_InfoTrack()
{
WebScrapper ws = new WebScrapper();
var results = await ws.GetMatchingURLs("test", Core.Enums.SearchEngine.Bing, new Uri("https://www.infotrack.com.au"));
Assert.IsTrue(results.Any());
}
public static SouqCities GetCities()
{
retry:
SyncSouqCookies();
var url = $"https://egypt.souq.com/eg-en/item_one.php";
try
{
var obj = WebScrapper.PostDownloadJson <SouqCities>(url, headers: new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
},
parameters: new Dictionary <string, string>()
{
{ "action", "get_cities" },
{ "country_code", "eg" },
{ "selected_city", "CAI" }
});
return(obj);
}
catch (HtmlScapperException ex)
{
goto retry;
}
}
private void MainForm_Load(object sender, EventArgs e)
{
this.ClientSize = Properties.Settings.Default.formSize;
Visibility.Update();
progressMain.ProgressText = "Updating Name Lookups from abgx.net";
_scanner = new GameScanner();
_scanner.OnGameProcessed += _loader_OnGameScanned;
_scanner.OnStatusUpdate += _loader_OnStatusUpdate;
_scanner.OnFinish += _loader_OnScanFinish;
_scrapper = new WebScrapper();
_scrapper.OnGameProcessed += _OnGameProcessed;
_scrapper.OnStatusUpdate += _loader_OnStatusUpdate;
_scrapper.OnFinish += _loader_OnFinish;
_abgx = new AbgxChecker();
_abgx.OnFinish += _loader_OnFinish;
_abgx.OnGameProcessed += _OnGameProcessed;
_abgx.OnStatusUpdate += _loader_OnStatusUpdate;
_move = new MoveProcessor();
_move.OnFinish += _loader_OnFinish;
_move.OnGameProcessed += _OnGameProcessed;
_move.OnStatusUpdate += _loader_OnStatusUpdate;
_info = new ScannerInfo();
progressMain.ProgressText = "";
progressSub.ProgressText = "";
if (Program.Wizard)
{
menuSettings_Click(null, null);
}
}
public eAlertProcess()
{
_serviceName = "eAlert Resender";
scrapper = new WebScrapper();
scrapper.baseUrl = ConfigurationManager.AppSettings["TeleCAREWebsite"];
five9URL = ConfigurationManager.AppSettings["Five9URL"];
SendEmailTo = ConfigurationManager.AppSettings["SendMailTo"];
}
public static IEnumerable <SouqSellerReview> GetSellerReview(string sellerId, int page = 0, int pageLimit = -1)
{
List <SouqSellerReview> tmpList = new List <SouqSellerReview>();
int tmpPage = 0;
bool stillAvailable;
do
{
retry:
SyncSouqCookies();
var url =
$"https://egypt.souq.com/eg-en/{sellerId.ToPercentageEncoding(SouqApiConstants.PercentageEncodingLevel)}/p/profile.html";
try
{
var obj = WebScrapper.PostDownloadJson <SouqSellerReview>(url, headers: new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
}, parameters: new Dictionary <string, string>()
{
{ "action", "show_more" },
{ "source", "seller_profile" },
{ "row", $"{tmpPage}" },
{ "seller", $"{sellerId}" },
{ "filter", "" },
{ "r_url", $"{url}" }
});
tmpList.Add(obj);
stillAvailable = obj.bShowMore;
if (pageLimit < 0 || page < page + pageLimit)
{
tmpPage++;
}
else
{
stillAvailable = false;
}
}
catch (HtmlScapperException ex)
{
/*Console Write*/
goto retry;
}
} while (stillAvailable);
return(tmpList);
}
public static HtmlDocument GetSellerProfile(string sellerId)
{
retry:
var url =
$"https://egypt.souq.com/eg-en/{sellerId.ToPercentageEncoding(SouqApiConstants.PercentageEncodingLevel)}/p/profile.html";
SyncSouqCookies();
try
{
var document = WebScrapper.GetDownloadHtml(url);
return(document);
}
catch (HtmlScapperException ex)
{
goto retry;
}
}
public static HtmlDocument GetCategories()
{
retry:
var url =
$"https://egypt.souq.com/eg-en/shop-all-categories/c/?ref=nav";
SyncSouqCookies();
try
{
var document = WebScrapper.GetDownloadHtml(url);
return(document);
}
catch (HtmlScapperException ex)
{
goto retry;
}
}
public static IEnumerable <SouqProductReview> GetProductReview(string productId, int page = 0, int pageLimit = -1)
{
List <SouqProductReview> tmpList = new List <SouqProductReview>();
int tmpPage = page;
bool stillAvailable;
do
{
retry:
SyncSouqCookies();
var url = $"https://egypt.souq.com/eg-en/reviews.php?page={tmpPage + 1}&action=get_reviews&id_item={productId}";
try
{
var obj = WebScrapper.GetDownloadJson <SouqProductReview>(url, headers: new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
});
tmpList.Add(obj);
stillAvailable = obj.show_more;
if (pageLimit < 0 || page < page + pageLimit)
{
tmpPage++;
}
else
{
stillAvailable = false;
}
}
catch (HtmlScapperException ex)
{
/*Console Write*/
goto retry;
}
} while (stillAvailable);
return(tmpList);
}
public void Run()
{
var credentialSection = Program.configuration.GetSection("Credentials");
var credentials = credentialSection.GetChildren().ToArray();
// Extraction and Transform
if (m_optionsCurrent != null)
{
scrapper = new WebScrapper(m_optionsCurrent.Flight, credentials[0].Value, credentials[1].Value, "https://www.flightradar24.com/21.33,-94.49/6", false);
}
else if (m_optionsHistorical != null)
{
scrapper = new WebScrapper(m_optionsHistorical.Flight, credentials[0].Value, credentials[1].Value, "https://www.flightradar24.com/21.33,-94.49/6", true);
}
if (scrapper.Run())
{
LoadDataIntoDatabase();
}
}
/// <summary>
/// Creates a web scrapper for provided <paramref name="url"/> and retrieves website's info.
/// </summary>
public static async Task <IApiResponse> GetWebsiteInfo(string url)
{
WebScrapper webScrapper;
try {
webScrapper = await WebScrapper.Create(url);
} catch (BadUrlException) {
return(new ErrorResponse {
Error = "Bad url."
});
}
var finalUrl = webScrapper.GetWebsiteUrl();
var title = webScrapper.GetWebsiteTitle();
var response = new UrlSolverResponse {
Url = finalUrl,
Title = title,
};
var notes = new List <string>();
if (finalUrl == url)
{
notes.Add("There was no redirection or redirection is made outside HTTP header.");
}
if (title is null)
{
notes.Add("Title is probably dynamically set after the page loads.");
}
if (notes.Count > 0)
{
response.Notes = notes;
}
return(response);
}
private static void SyncSouqCookies()
{
if (ExpireSession > DateTime.Now)
{
return;
}
var url = "https://egypt.souq.com/eg-en/shop-all-categories/c/?ref=nav";
retry:
try
{
var obj = (WebScrapper.GetHeaders(url))["Set-Cookie"];
var cc = GetAllCookiesFromHeader(obj, url);
ExpireSession = DateTime.Now.AddYears(1).AddDays(-2);
Cookies = cc;
cookieString = obj;
}
catch (Exception ex)
{
goto retry;
}
}
public PhysicianStatusProcessor()
{
_serviceName = "Physician Status Mover Service";
scrapper = new WebScrapper();
scrapper.baseUrl = ConfigurationManager.AppSettings["TeleCAREWebsite"];
}
public void NaiveMouser_Test3()
{
Assert.AreEqual("Thick Film Resistors - SMD 2.2K OHM 1% Learn More", WebScrapper.NaiveMouser("RC0603FR-072K2L"));
}
public void NaiveMouser_Test2()
{
Assert.AreEqual("Ferrite Beads 220 OHM HIGH SPEED SIGNAL Learn More", WebScrapper.NaiveMouser("BLM15BD221SN1D"));
}
public void NaiveMouser_Test()
{
Assert.AreEqual("Multilayer Ceramic Capacitors MLCC - SMD/SMT 0402 4.7uF 6.3volts *Derate Voltage/Temp Learn More", WebScrapper.NaiveMouser("GRM155R60J475ME87D"));
}
public void NaiveElement14_Test2()
{
Assert.AreEqual("MURATA - Ferrite Bead, 0402 [1005 Metric], 220 ohm, 300 mA, BLM15B Series, 0.4 ohm, ± 25%", WebScrapper.NaiveElement14("BLM15BD221SN1D"));
}
public void NaiveElement14_Test()
{
Assert.AreEqual("YAGEO - SMD Chip Resistor, 2.2 kohm, RC Series, 75 V, Thick Film, 0603 [1608 Metric], 100 mW ", WebScrapper.NaiveElement14("RC0603FR-072K2L"));
}
public void NaiveRS_Test3()
{
Assert.AreEqual("Murata 10nF Multilayer Ceramic Capacitor MLCC 6.3V dc ±10% X7R Dielectric 0402 (1005M) SMD, Max. Temp. +125°C ", WebScrapper.NaiveRS("GRM155R70J103KA01D"));
}
public void NaiveRS_Test2()
{
Assert.AreEqual("Murata Ferrite Bead (Chip Ferrite Bead), 1 x 0.5 x 0.5mm (0402 (1005M)), 220Ω impedance at 100 MHz ", WebScrapper.NaiveRS("BLM15BD221SN1D"));
}
public void NaiveRS_Test()
{
Assert.AreEqual("TE Connectivity CR2032 PCB Battery Holder, Leaf Spring Contact ", WebScrapper.NaiveRS("796136-1"));
}
public void NaiveDigikey_Test2()
{
Assert.AreEqual("DIODE ARRAY SCHOTTKY 30V SOT23 ", WebScrapper.NaiveDigikey("BAT54S, 215"));
}
public void NaiveDigikey_Test()
{
Assert.AreEqual("CAP CER 1000PF 100V C0G 0603 ", WebScrapper.NaiveDigikey("C1608C0G2A102J080AA"));
}
public void Element14API_GetDescriptionTest()
{
Assert.AreEqual("ABRACON - ABM3B-12.000MHZ-B2-T - CRYSTAL, 12MHZ, 18PF, 5 X 3.2MM", WebScrapper.Element14API_GetDescription("ABM3B-12.000MHZ-B2-T"));
}
public void NaiveMouser_Test4()
{
Assert.AreEqual("Crystals +/-20ppm 12MHZ", WebScrapper.NaiveMouser("ABM3B-12.000MHZ-B2-T"));
}
public void API_MouserTest()
{
Assert.AreEqual("Crystals +/-20ppm 12MHZ", WebScrapper.MouserAPI_GetDescription("ABM3B-12.000MHZ-B2-T"));
}
public async Task CallURL_should_fire_exception_for_invalid_url()
{
var result = await WebScrapper.CallUrl("https://www.vj.au");
}
