public static SouqDeliveryInfo GetDeliveryInfo(string cityCode, string cityName, string unitId, string hitsCfs, string hitsCfsMeta)
{
retry:
SyncSouqCookies();
var url =
$"https://egypt.souq.com/eg-en/item_one.php?action=estimated_delivery¤tCityISO={cityCode}¤tCity={cityName}&" +
$"idUnit={unitId}&hitsCfs={hitsCfs}&hitsCfsMeta={hitsCfsMeta.ToPercentageEncoding(1)}";
try
{
var obj = WebScrapper.GetDownloadJson <SouqDeliveryInfo>(url, headers: new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
}, cookies: Cookies);
return(obj);
}
catch (HtmlScapperException ex)
{
if (ex.InnerException.Message.Contains("Too many automatic redirections were attempted"))
{
throw;
}
goto retry;
}
}
public static IEnumerable <SouqProductReview> GetProductReview(string productId, int page = 0, int pageLimit = -1)
{
List <SouqProductReview> tmpList = new List <SouqProductReview>();
int tmpPage = page;
bool stillAvailable;
do
{
retry:
SyncSouqCookies();
var url = $"https://egypt.souq.com/eg-en/reviews.php?page={tmpPage + 1}&action=get_reviews&id_item={productId}";
try
{
var obj = WebScrapper.GetDownloadJson <SouqProductReview>(url, headers: new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
});
tmpList.Add(obj);
stillAvailable = obj.show_more;
if (pageLimit < 0 || page < page + pageLimit)
{
tmpPage++;
}
else
{
stillAvailable = false;
}
}
catch (HtmlScapperException ex)
{
/*Console Write*/
goto retry;
}
} while (stillAvailable);
return(tmpList);
}
public static IEnumerable <SouqProductsGrid2> SearchProducts(string categoryId)
{
List <SouqProductsGrid2> tmpList = new List <SouqProductsGrid2>();
int section = 1;
int page = 1;
bool stillAvailable;
do
{
retry:
SyncSouqCookies();
var url = $"https://egypt.souq.com/eg-en/{categoryId}/s/?page={page}§ion={section}&action=grid";
//Console.WriteLine(url);
try
{
var obj = WebScrapper.GetDownloadJson <SouqProductsGrid2>(url, headers: new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
});
obj.page = page;
obj.section = section;
if (string.IsNullOrEmpty(obj.redirect_url))
{
/*Under Test Fields*/
var is_ags_test = obj.jsonData.units.Where(l => l.is_ags);
if (is_ags_test.Any())
{
Debugger.Break();
}
var is_price_cbt = obj.jsonData.units.Where(l => l.price_cbt != "");
if (is_price_cbt.Any())
{
Debugger.Break();
}
var is_coupon = obj.jsonData.units.Where(l => l.coupon != null && l.coupon.ToString() != "[]");
if (is_coupon.Any())
{
Debugger.Break();
}
var is_InternationalSeller = obj.jsonData.units.Where(l => l.isInternationalSeller);
if (is_InternationalSeller.Any())
{
Debugger.Break();
}
var is_shipping = obj.jsonData.units.Where(l => l.shipping != false);
if (is_shipping.Any())
{
Debugger.Break();
}
var is_selling_points = obj.jsonData.units.Where(l => l.selling_points.ToString() != "False");
if (is_selling_points.Any())
{
Debugger.Break();
}
}
else
{
//Debugger.Break();
categoryId = obj.redirect_url.ExtractCategoryId();
goto retry;
}
tmpList.Add(obj);
stillAvailable = obj.show_more == 1;
if (section == 2)
{
page++;
section = 1;
}
else
{
section++;
}
}
catch (HtmlScapperException ex)
{
/*Console Write*/
Console.WriteLine(DateTime.Now + "-" + ex.InnerException?.Message ?? ex.Message);
goto retry;
}
} while(stillAvailable);
return(tmpList.ToList());
}
public static IEnumerable <SouqProductsGrid2> SearchProductsParallel(string categoryId)
{
ConcurrentBag <SouqProductsGrid2> tmpList = new ConcurrentBag <SouqProductsGrid2>();
int requestCount = 0;
retry_peek:
try
{
SyncSouqCookies();
var peek_url = $"https://egypt.souq.com/eg-en/{categoryId}/s/?page=1§ion=1&action=grid";
var peek_obj = WebScrapper.GetDownloadJson <SouqProductsGrid2>(peek_url,
headers: new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
});
if (!string.IsNullOrEmpty(peek_obj.redirect_url))
{
categoryId = peek_obj.redirect_url.ExtractCategoryId();
goto retry_peek;
}
requestCount = (int)Math.Ceiling((float)peek_obj.jsonData.meta_data.total / SouqApiConstants.ProductSectionListLimit);
}
catch (Exception ex)
{
/*Console Write*/
Console.WriteLine(DateTime.Now + "-" + ex.InnerException?.Message ?? ex.Message);
goto retry_peek;
}
var pagingGenerator = new PagingLinkGenerator(requestCount);
Parallel.ForEach(pagingGenerator, (pageInfo) =>
{
retry:
SyncSouqCookies();
var url = $"https://egypt.souq.com/eg-en/{categoryId}/s/?page={pageInfo.Page}§ion={pageInfo.Section}&action=grid";
//Console.WriteLine(url);
try
{
var obj = WebScrapper.GetDownloadJson <SouqProductsGrid2>(url,
headers: new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
});
/*Under Test Fields*/
var is_ags_test = obj.jsonData.units.Where(l => l.is_ags);
if (is_ags_test.Any())
{
Debugger.Break();
}
var is_price_cbt = obj.jsonData.units.Where(l => l.price_cbt != "");
if (is_price_cbt.Any())
{
Debugger.Break();
}
var is_coupon = obj.jsonData.units.Where(l => l.coupon != null && l.coupon.ToString() != "[]");
if (is_coupon.Any())
{
Debugger.Break();
}
var is_InternationalSeller = obj.jsonData.units.Where(l => l.isInternationalSeller);
if (is_InternationalSeller.Any())
{
Debugger.Break();
}
var is_shipping = obj.jsonData.units.Where(l => l.shipping != false);
if (is_shipping.Any())
{
Debugger.Break();
}
var is_selling_points = obj.jsonData.units.Where(l => l.selling_points.ToString() != "False");
if (is_selling_points.Any())
{
Debugger.Break();
}
tmpList.Add(obj);
}
catch (Exception ex)
{
/*Console Write*/
Console.WriteLine(DateTime.Now + "-" + ex.InnerException?.Message ?? ex.Message);
goto retry;
}
});
return(tmpList.ToList());
}
public static IEnumerable <SouqProductsGrid> GetProductsFromCategory(string categoryId)
{
List <SouqProductsGrid> tmpList = new List <SouqProductsGrid>();
int section = 1;
int page = 1;
bool stillAvailable;
peek_retry:
SyncSouqCookies();
try
{
var url_peek = $"https://egypt.souq.com/eg-en/{categoryId}/l/?ref=nav&page=1§ion=1";
var new_url = WebScrapper.GetResponseUrl(url_peek);
if (!new_url.Equals(url_peek))
{
return(null);
}
}
catch
{
goto peek_retry;
}
do
{
retry:
SyncSouqCookies();
var url = $"https://egypt.souq.com/eg-en/{categoryId}/l/?ref=nav&page={page}§ion={section}&action=grid";
try
{
var obj = WebScrapper.GetDownloadJson <SouqProductsGrid>(url, headers: new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
});
tmpList.Add(obj);
stillAvailable = obj.show_more == 1;
if (section == 2)
{
page++;
section = 1;
}
else
{
section++;
}
}
catch (HtmlScapperException ex)
{
/*Console Write*/
goto retry;
}
} while (stillAvailable);
return(tmpList);
}
public static SouqProduct GetProduct(string productFullId)
{
retry:
SyncSouqCookies();
var url = $"https://egypt.souq.com/eg-en/{productFullId}/i/";
try
{
var obj = WebScrapper.GetDownloadJson(url, headers: new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
});
SouqProduct product = null;
if (obj != null)
{
product = new SouqProduct
{
body = obj.Property("body").Value.ToString(),
bundles = null
};
if (obj.Property("aBundles").Value.ToString() != "[]")
{
var bundle = obj.Property("aBundles").Value.ToObject <JObject>();
var bundlesList = bundle.Properties().Values().Select(l => l.ToObject <JObject>()).ToList();
product.bundles = new List <SouqBundle>();
foreach (var tmpBundle in bundlesList)
{
var bundleObj = tmpBundle.ToObject <SouqBundle>();
if (tmpBundle.Property("units") != null)
{
var units = tmpBundle.Property("units").Value.ToObject <JObject>();
var unitsList = units.Properties().Values().Select(l => l.ToObject <JObject>()).ToList();
bundleObj.units = unitsList.Select(l => l.ToObject <SouqBundleUnits>()).ToList();
}
product.bundles.Add(bundleObj);
}
}
}
else
{
var objHtml = WebScrapper.GetDownloadHtml(url);
product = new SouqProduct
{
body = objHtml.DocumentNode.OuterHtml,
bundles = null
};
}
return(product);
}
catch (HtmlScapperException ex)
{
goto retry;
}
}