private void GenerateRealSize(ProductDetails resultDetails, string html, string caster)
{
var ind = html.IndexOf("-", StringComparison.Ordinal);
var before = html.Substring(0, ind != -1 ? ind : html.Length).Trim();
var after = html.Substring(ind != -1 ? ind + 1 : html.Length).Trim();
after = after.Length > 0 ? after : "Unknown";
var result = before;
if (int.TryParse(before, out var val))
{
var index = caster.IndexOf(val.ToString(), StringComparison.Ordinal);
if (index == -1)
{
resultDetails.AddSize(before, after);
return;
}
var indOfEqualitySign = caster.IndexOf("=", index, StringComparison.Ordinal);
var indOfTokenFinish = caster.IndexOf(",", indOfEqualitySign, StringComparison.Ordinal);
if (indOfTokenFinish == -1)
{
indOfTokenFinish = caster.Length;
}
result = caster.Substring(indOfEqualitySign + 1, indOfTokenFinish - indOfEqualitySign - 1).Trim();
}
resultDetails.AddSize(result, after);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var client = ClientFactory.GetProxiedFirefoxClient();
var doc = client.GetDoc(productUrl, token);
var page = doc.DocumentNode;
var infoContainer = page.SelectSingleNode("//div[@id = 'pchange-target']/div");
var collectionName =
infoContainer.SelectSingleNode("./span[@itemprop = 'name']").InnerHtml.EscapeNewLines();
var productName = infoContainer.SelectSingleNode("./h1[@itemprop = 'name']").InnerHtml.EscapeNewLines();
var name = $"{collectionName} - {productName}";
var priceNode = infoContainer.SelectSingleNode("./div[contains(@class, 'product-price')]/span");
var price = Utils.ParsePrice(priceNode.InnerHtml);
var image = WebsiteBaseUrl + page.SelectSingleNode("//img[contains(@class, 'primary-image')]")
.GetAttributeValue("src", "").Substring(1);
var details = new ProductDetails
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
details.AddSize(productName, "Unknown");
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
HtmlDocument doc = new HtmlDocument();
var client = ClientFactory.GetProxiedFirefoxClient();
var response = CfBypasser.GetRequestedPage(client, this, productUrl, token);
doc.LoadHtml(response.Content.ReadAsStringAsync().Result);
var root = doc.DocumentNode;
var sizeNodes = root.SelectNodes("//*[contains(@class,'styled-radio')]/label");
var sizes = sizeNodes.Select(node => node.InnerText).ToList();
var name = root.SelectSingleNode("//*[contains(@class, 'prod-title')]").InnerText.Trim();
var priceNode = root.SelectSingleNode("//div[contains(@class, 'price')]/span/strong");
var price = Utils.ParsePrice(priceNode.InnerText);
var image = root.SelectSingleNode("//*[@id='image-0']").GetAttributeValue("src", null);
ProductDetails result = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
foreach (var size in sizes)
{
result.AddSize(size, "Unknown");
}
return(result);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
var name = document.SelectSingleNode("//div[contains(@class, 'product_info')]/h1").InnerHtml;
var image = WebsiteBaseUrl +
document.SelectSingleNode("//div[@id='large_img']/img").GetAttributeValue("src", "");
var priceNode = document.SelectSingleNode("//div[contains(@class, 'price')]").InnerHtml;
var price = Utils.ParsePrice(priceNode);
var details = new ProductDetails
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
var node = document.SelectSingleNode("//*[@id='addToCart']/div/div/div/select[@id = 'size']");
var sizeCollection = node.SelectNodes("./option");
foreach (var size in sizeCollection)
{
var sz = size.GetAttributeValue("value", "");
if (sz.Length > 0)
{
details.AddSize(sz, "Unknown");
}
}
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
HtmlNode ds = GetWebpage(productUrl, token);
string name = ds.SelectSingleNode("//h1[contains(@data-auto-id,'product-title')]").InnerText;
string priceIntoString = ds.SelectSingleNode("//div[contains(@class,'gl-price-container')]/span").InnerText;
string result = Regex.Match(priceIntoString, @"[\d\.]+").Value;
double.TryParse(result, NumberStyles.Any, CultureInfo.InvariantCulture, out var price);
string imageURL = ds.SelectSingleNode("//img[@class='performance-item']").GetAttributeValue("src", null);
int t = productUrl.LastIndexOf("/", StringComparison.Ordinal);
int r = productUrl.LastIndexOf(".", StringComparison.Ordinal);
var sku = productUrl.Substring(t + 1, r - t - 1);
ProductDetails details = new ProductDetails()
{
Name = name,
Price = price,
ImageUrl = imageURL,
Url = productUrl,
Id = productUrl,
Currency = "CHF",
ScrapedBy = this
};
string search = $"https://www.adidas.com/api/products/{sku}/availability?sitePath=us";
ds = GetWebpage(search, token);
string toJason = ds.InnerText;
int arrayStart = toJason.LastIndexOf("[", StringComparison.Ordinal);
int arrayEnd = toJason.LastIndexOf("}");
if (arrayStart == -1)
{
return(details);
}
toJason = toJason.Substring(arrayStart, arrayEnd - arrayStart);
JArray parsed = null;
try
{
parsed = JArray.Parse(toJason);
}
catch (Exception e)
{
return(details);
}
foreach (var x in parsed.Children())
{
var value = (string)x.SelectToken("size");
var availability_status = (string)x.SelectToken("availability_status");
if (availability_status == "IN_STOCK")
{
details.AddSize(value, "Unknown");
}
}
return(details);
}
private void addSizes(JObject main, ProductDetails productDetails)
{
JArray sizeArr = JArray.Parse(main["AVAILABLE_SIZES"].ToString());
foreach (var elem in sizeArr)
{
productDetails.AddSize(elem.ToString(), "Unknown");
}
}
private void addSizes(HtmlNode document, ProductDetails productDetails)
{
HtmlNodeCollection liNodes = document.SelectNodes("//*[@id=\"itemSizes\"]/ul/li");
foreach (var liNode in liNodes)
{
productDetails.AddSize(Utils.EscapeNewLines(liNode.InnerText), "Unknown");
}
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
var price = Utils.ParsePrice(document.SelectSingleNode("//p[@class='Price']/span/span[1]").InnerHtml);
string name = document.SelectSingleNode("//h1[@class='Title']").InnerText.Trim();
string image = document.SelectSingleNode("//div[@class='imgs']/div/a/img").GetAttributeValue("src", "");
name = Regex.Replace(name, @"\s+", " ");
string brand = null;
if (document.SelectSingleNode("//span[@id='Brand-Title']") != null)
{
brand = document.SelectSingleNode("//span[@class='Brand-Title']").InnerText;
}
ProductDetails details = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency.Replace("€", "EUR"),
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this,
BrandName = brand
};
var sizeCollection = document.SelectNodes("//select[@name='id']/option");
if (sizeCollection != null)
{
foreach (var size in sizeCollection)
{
if (!size.InnerText.Contains("Out of Stock"))
{
string sz = size.InnerHtml;
if (sz.Contains("Select"))
{
continue;
}
if (sz.Length > 0)
{
details.AddSize(sz, "Unknown");
}
}
}
}
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
Price price;
if (document.SelectSingleNode("//ins/span[@class='woocommerce-Price-amount amount']") != null)
{
price = Utils.ParsePrice(document.SelectSingleNode("//ins/span[@class='woocommerce-Price-amount amount']").InnerText.Replace(",", ".").Replace(" ", ""));
}
else
{
price = Utils.ParsePrice(document.SelectSingleNode("//span[@class='woocommerce-Price-amount amount']").InnerText.Replace(",", ".").Replace(" ", ""));
}
string name = document.SelectSingleNode("//h3[@class='product_title']").InnerText.Trim();
string image = document.SelectSingleNode("//div[@class='swiper-slide']/img").GetAttributeValue("src", "");
string brand = null;
if (document.SelectSingleNode("//meta[@property='og:brand']") != null)
{
brand = document.SelectSingleNode("//meta[@property='og:brand']").GetAttributeValue("content", null);
}
ProductDetails details = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency.Replace("€", "EUR"),
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
var sizeCollection = document.SelectNodes("//td[@class='value']/label");
foreach (var size in sizeCollection)
{
string sz = size.InnerHtml;
if (sz.Length > 0)
{
details.AddSize(sz, "Unknown");
}
}
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
var price = Utils.ParsePrice(document.SelectSingleNode("//span[@id='js-product-price']/span[@class='product-content__price--inc']/span").InnerText.Replace(",", "."));
string name = document.SelectSingleNode("//span[@id='js-product-title']").InnerText.Trim();
string image = WebsiteBaseUrl + document.SelectSingleNode("//img[@id='js-product-main-image']").GetAttributeValue("data-src", "");
string brand = null;
if (document.SelectSingleNode("//span[@class='product-content__title--brand']") != null)
{
brand = document.SelectSingleNode("//span[@class='product-content__title--brand']").InnerText;
}
ProductDetails details = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency.Replace("€", "EUR"),
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
string id = productUrl.Substring(productUrl.Length - 5);
string restApiUrl = "https://www.triads.co.uk/ajax/get_product_options/" + id;
var client = ClientFactory.GetProxiedFirefoxClient(autoCookies: true);
var response = Utils.GetParsedJson(client, restApiUrl, token);
foreach (var item in response["attributes"])
{
if (item["name"].ToString() == "UK Size")
{
foreach (var value in item["values"])
{
if (int.Parse(value["stock_level"].ToString()) > 0)
{
details.AddSize(value["value"].ToString(), value["stock_level"].ToString());
}
}
}
}
return(details);
}
private ProductDetails DropDownSizesList(HtmlNode sizesNode, ProductDetails details)
{
var sizes = sizesNode.SelectNodes("./option");
foreach (var size in sizes)
{
if (size.GetAttributeValue("class", null) != null && size.GetAttributeValue("class", null) != "disabled")
{
details.AddSize(ExtractEuroSize(size.InnerText.Trim()), "Unknown");
}
}
return(details);
}
private ProductDetails OnScreenSizesList(HtmlNode sizesNode, ProductDetails details)
{
var sizes = sizesNode.SelectNodes("./li");
foreach (var size in sizes)
{
if (size.SelectSingleNode("./a[contains(@class,'disabled')]") == null)
{
details.AddSize(size.InnerText.Trim(), "Unknown");
}
}
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
var price = Utils.ParsePrice(document.SelectSingleNode("//span[@class='product_price']").InnerHtml);
string name = document.SelectSingleNode("//span[@class='name']").InnerText.Trim();
string image = document.SelectSingleNode("//img[@id='main-image']").GetAttributeValue("src", "");
string brand = null;
if (document.SelectSingleNode("//span[@class='brand']") != null)
{
brand = document.SelectSingleNode("//span[@class='brand']").InnerText;
}
ProductDetails details = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency.Replace("€", "EUR"),
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this,
BrandName = brand
};
var sizeCollection = document.SelectNodes("//div[@class='psizeoptioncontainer']/div/a");
if (sizeCollection != null)
{
foreach (var size in sizeCollection)
{
if (!size.GetAttributeValue("class", "").Contains("piunavailable"))
{
string sz = size.InnerHtml;
if (sz.Length > 0)
{
details.AddSize(sz, "Unknown");
}
}
}
}
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
string innerHtml = document.InnerHtml;
int startIndx = document.InnerHtml.IndexOf("\"size\"" + ":[", StringComparison.Ordinal);
if (startIndx == -1)
{
return(null);
}
int endIndx = -1;
endIndx = innerHtml.IndexOf("]", startIndx, StringComparison.Ordinal);
if (endIndx == -1)
{
return(null);
}
string jsonObjectStr = innerHtml.Substring(startIndx, endIndx - startIndx + 1);
jsonObjectStr = jsonObjectStr.Substring(jsonObjectStr.IndexOf("[", StringComparison.Ordinal));
JArray parsed = JArray.Parse(jsonObjectStr);
string name = document.SelectSingleNode("//div[contains(@class, 'Z22ltwr')]/h1").InnerText;
string priceIntoString = document.SelectSingleNode("//span[contains(@class, 'currentPriceString_PYXT2')]").InnerText;
string result = Regex.Match(priceIntoString, @"[\d\.]+").Value;
double.TryParse(result, NumberStyles.Any, CultureInfo.InvariantCulture, out var price);
string imageURL = document.SelectSingleNode("//img[contains(@class, 'mainImage_Z2iFhqF')]").GetAttributeValue("src", null);
ProductDetails details = new ProductDetails()
{
Name = name,
Price = price,
ImageUrl = imageURL,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
foreach (var x in parsed.Children())
{
var value = (string)x.SelectToken("displayValue");
details.AddSize(value, "Unknown");
}
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
if (document == null)
{
Logger.Instance.WriteErrorLog($"Can't Connect to overkill website");
throw new WebException("Can't connect to website");
}
var root = document.DocumentNode;
var name = root.SelectSingleNode("//div[@class='row-fluid product-name']/div[contains(@class, 'span')]/h2")?.InnerText.Trim().Replace("<br>", "\n");
var priceNode = root.SelectSingleNode("//div[contains(@class, 'price-box')]//span[@class='price']");
var price = Utils.ParsePrice(priceNode?.InnerText);
var image = root.SelectSingleNode("//meta[@property='og:image']")?.GetAttributeValue("content", null);
ProductDetails result = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
if (!root.InnerHtml.Contains("new Product.Config"))
{
return(result);
}
var jsonStr = Regex.Match(root.InnerHtml, @"var spConfig = new Product.Config\((.*)\)").Groups[1].Value;
var tokenStr = Regex.Match(jsonStr, "\"(\\d+)\":").Groups[1].Value;
JObject parsed = JObject.Parse(jsonStr);
var sizes = parsed.SelectToken("attributes").SelectToken(tokenStr).SelectToken("options");
foreach (JToken sz in sizes.Children())
{
var sizeName = (string)sz.SelectToken("label");
JArray products = (JArray)sz.SelectToken("products");
if (products.Count > 0)
{
result.AddSize(sizeName, "Unknown");
}
}
return(result);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
if (document == null)
{
Logger.Instance.WriteErrorLog($"Can't Connect to endclothing website");
throw new WebException("Can't connect to website");
}
var root = document.DocumentNode;
var name = root.SelectSingleNode("//h1[@class='page-title']/span")?.InnerText.Trim();
var priceNode = root.SelectSingleNode("//span[@class='price'][last()]");
var price = Utils.ParsePrice(priceNode?.InnerText);
var image = root.SelectSingleNode("//img[@class='swiper-slide']")?.GetAttributeValue("src", null);
ProductDetails result = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
if (!root.InnerHtml.Contains("spConfig"))
{
return(result);
}
var jsonStr = Regex.Match(root.InnerHtml, "\"spConfig\": (.*?),\n").Groups[1].Value;
var tokenStr = Regex.Match(jsonStr, "\"(\\d+)\":").Groups[1].Value;
JObject parsed = JObject.Parse(jsonStr);
var sizes = parsed.SelectToken("attributes").SelectToken(tokenStr).SelectToken("options");
foreach (JToken sz in sizes.Children())
{
var sizeName = (string)sz.SelectToken("label");
JArray products = (JArray)sz.SelectToken("products");
if (products.Count > 0)
{
result.AddSize(sizeName, "Unknown");
}
}
return(result);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
Price price;
if (document.SelectSingleNode("//div[@class='product-price']/span[@class='sale']") != null)
{
price = Utils.ParsePrice(document.SelectSingleNode("//div[@class='product-price']/span[@class='sale']").InnerText);
}
else
{
price = Utils.ParsePrice(document.SelectSingleNode("//div[@class='product-price']/span[@class='price']").InnerText);
}
string name = document.SelectSingleNode("//h1[@id='product-name']").InnerText.Trim();
string image = WebsiteBaseUrl + document.SelectSingleNode("//img[@id='primary-image']").GetAttributeValue("src", "");
ProductDetails details = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency.Replace("&EURO;", "EUR"),
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
var sizeCollection = document.SelectNodes("//span[@class='size-type']");
if (sizeCollection != null)
{
foreach (var size in sizeCollection)
{
string sz = size.GetAttributeValue("title", size.InnerText).Trim();
if (sz.Length > 0)
{
details.AddSize(sz, "Unknown");
}
}
}
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
Price price;
if (document.SelectSingleNode("//h1[@class='price']/span[@class='sale']/span") != null)
{
price = Utils.ParsePrice(document.SelectSingleNode("//h1[@class='price']/span[@class='sale']/span").InnerText);
}
else
{
price = Utils.ParsePrice(document.SelectSingleNode("//h1[@class='price']").InnerText);
}
string name = document.SelectSingleNode("//h1[not(@class='price')]").InnerText;
string image = document.SelectSingleNode("//div[@class='product-gallery']/img[1]").GetAttributeValue("src", "");
ProductDetails details = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
var sizeCollection = document.SelectNodes("//select[@name='id']/option");
if (sizeCollection != null)
{
foreach (var size in sizeCollection)
{
string sz = size.InnerHtml.Trim();
if (sz.Contains("Sold out"))
{
continue;
}
if (sz.Length > 0)
{
details.AddSize(sz, "Unknown");
}
}
}
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
if (document == null)
{
Logger.Instance.WriteErrorLog($"Can't Connect to basketrevolution website");
throw new WebException("Can't connect to website");
}
var product = document.SelectSingleNode("//div[contains(@class, 'product--detail-upper')]");
var imgurl = product
.SelectSingleNode(
"//div[contains(@class,'image--box')][1]/span[1]/span[@class='image--media'][1]/img[1]")
.GetAttributeValue("srcset", null);
var name = product.SelectSingleNode("//h1[@class='product--title'][1]/span[1]").InnerHtml;
var price = Utils.ParsePrice(product.SelectSingleNode("//div[@class='w_price-infos'][1]").InnerHtml);
var sizes = product.SelectSingleNode("//select[contains(@class,'w_select-item')][1]");
var sizesList = sizes.SelectNodes("./option");
ProductDetails result = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = imgurl,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
foreach (var size in sizesList)
{
var sizeString = size.InnerHtml;
if (sizeString != "size")
{
result.AddSize(sizeString, "Unknown");
}
}
return(result);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
var price = Utils.ParsePrice(document.SelectSingleNode("//p[@class='our_price_display']/span[@class='price']").InnerHtml);
string name = document.SelectSingleNode("//h1").InnerText;
string image = document.SelectSingleNode("//div[@class='owl-carousel']/div/img").GetAttributeValue("data-src", "");
string brand = null;
if (document.SelectSingleNode("//a[contains(@class,'product__manufacturer')]") != null)
{
brand = document.SelectSingleNode("//a[contains(@class,'product__manufacturer')]").GetAttributeValue("title", null);
}
ProductDetails details = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this,
BrandName = brand
};
var sizeCollection = document.SelectNodes("//select[@id='size-select']/option");
if (sizeCollection != null)
{
foreach (var size in sizeCollection)
{
string sz = size.InnerHtml;
if (sz.Length > 0)
{
details.AddSize(sz, "Unknown");
}
}
}
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var resp = GetWebpage(productUrl, token).DocumentNode;
if (resp == null)
{
Logger.Instance.WriteErrorLog($"Can't Connect to basketrevolution website");
throw new WebException("Can't connect to website");
}
var product = resp.SelectSingleNode("//div[contains(@class, 'primary_block')]");
var name = product.SelectSingleNode("//h1[@class='h4'][1]").InnerText;
var price = Utils.ParsePrice(product.SelectSingleNode("//span[@id='our_price_display'][1]").InnerText);
var imgurl = product.SelectSingleNode("//div[contains(@class,'img-item')][1]/img[contains(@class,'lazyload')][1]").GetAttributeValue("src", null);
ProductDetails result = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = imgurl,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
var sizesStr = Regex.Match(resp.OuterHtml, @"var combinations=(.*?)}}").Groups[1].Value + "}}";
var sizes = JObject.Parse(sizesStr);
foreach (var size in sizes.Children())
{
var sizeVal = size.First.SelectToken("attributes_values").First.First.ToString();
var quantity = size.First.SelectToken("quantity").ToString();
if (int.Parse(quantity) > 0)
{
result.AddSize(sizeVal, quantity);
}
}
return(result);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
if (document == null)
{
Logger.Instance.WriteErrorLog($"Can't Connect to nighshop website");
throw new WebException("Can't connect to website");
}
var root = document.DocumentNode;
var name = root.SelectSingleNode("//div[@class='product-name']/h1")?.InnerText.Trim();
var priceNode = root.SelectSingleNode("//span[@class='price'][last()]");
var price = Utils.ParsePrice(priceNode?.InnerText);
var image = root.SelectSingleNode("//img[@class='owl-lazy']")?.GetAttributeValue("data-src", null);
ProductDetails result = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
var sizeCollection = root.SelectNodes("//div[contains(@class, 'attribute-item')]");
foreach (var size in sizeCollection)
{
string sz = size.InnerText.Trim();
if (sz.Length <= 0)
{
continue;
}
if (!(size.GetAttributeValue("class", null).Contains("disabled")))
{
result.AddSize(sz, "Unknown");
}
}
return(result);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var resp = GetWebpage(productUrl, token);
if (resp == null)
{
Logger.Instance.WriteErrorLog($"Can't Connect to basketrevolution website");
throw new WebException("Can't connect to website");
}
var product = resp.SelectSingleNode("//div[contains(@class, 'row col-wrap')]");
var name = product.SelectSingleNode("//h2[1]").InnerHtml;
var price = Utils.ParsePrice(product.SelectSingleNode("//p[contains(@class, 'price')][1]/b").InnerHtml);
var imgurl = "https://byparra.com" + product
.SelectSingleNode("//div[@class='item active'][1]/img[@class='img-responsive'][1]")
.GetAttributeValue("src", null);
var sizes = product.SelectSingleNode("//select[contains(@class,'form-control')][1]");
var sizesList = sizes.SelectNodes("//option");
ProductDetails result = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = imgurl,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
foreach (var size in sizesList)
{
var sizeString = size.InnerHtml;
if (sizeString != "size")
{
result.AddSize(sizeString, "Unknown");
}
}
return(result);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var client = ClientFactory.GetProxiedFirefoxClient(autoCookies: true);
GetWebpage(client, WebsiteBaseUrl, token);
var resp = GetWebpage(client, productUrl, token);
if (resp == null)
{
Logger.Instance.WriteErrorLog($"Can't Connect to ruvilla website");
throw new WebException("Can't connect to website");
}
var product = resp.SelectSingleNode("//section[contains(@class, 'product-details')]");
var name = product.SelectSingleNode("//h1[@class='product-title'][1]").InnerText;
var priceNode = product.SelectSingleNode("//div[@class='price-box']");
var price = Utils.ParsePrice(priceNode.SelectSingleNode("//span[contains(@id,'product')][1]").InnerText);
var imgurl = product.SelectSingleNode("//figure[1]/img[1]").GetAttributeValue("src", null);
ProductDetails result = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = imgurl,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
string pattern = @"var spConfig = new Product.Config\((.*?)\)";
var match = Regex.Match(resp.OuterHtml, pattern).Groups[1].Value;
var jObj = JObject.Parse(match);
var sizes = jObj.SelectToken("attributes").SelectToken("196").SelectToken("options").Children();
foreach (var size in sizes)
{
result.AddSize(size.Value <string>("label"), "Unknown");
}
return(result);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
var price = Utils.ParsePrice(document.SelectSingleNode("//span[@itemprop='price']").InnerText.Replace(",", "."));
string name = document.SelectSingleNode("//h3[@itemprop='name']").InnerText;
string image = document.SelectSingleNode("//li[@class='homeslider-container'][1]/img").GetAttributeValue("src", "");
string brand = null;
if (document.SelectSingleNode("//div/h2/a") != null)
{
brand = document.SelectSingleNode("//div/h2/a").InnerText;
}
ProductDetails details = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this,
BrandName = brand
};
var sizeCollection = document.SelectNodes("//select[@name='group_1']/option");
if (sizeCollection != null)
{
foreach (var size in sizeCollection)
{
string sz = size.InnerHtml;
if (sz.Length > 0)
{
details.AddSize(sz, "Unknown");
}
}
}
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var page = GetWebpage(productUrl, token);
var nameContainer = page.SelectSingleNode("//div[contains(@class, 'product-name')]/span");
var name = nameContainer.InnerHtml.EscapeNewLines();
var image = page.SelectSingleNode("//div[contains(@class, 'product-image')]/img")
.GetAttributeValue("src", "");
var priceNode = page.SelectSingleNode("//div[contains(@class, 'product-shop')]");
ProductDetails details = new ProductDetails()
{
Price = GetPrice(priceNode),
Name = name,
Currency = "$",
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
//product.ImageUrl = page.SelectSingleNode("//img[@id = 'image-main']").GetAttributeValue("src", null);
var jsonStr = Regex.Match(page.InnerHtml, @"var spConfig = new Product.Config\((.*)\)").Groups[1].Value;
JObject parsed = JObject.Parse(jsonStr);
var sizes = GetSizesToken(parsed);
sizes = sizes.SelectToken("options");
foreach (JToken sz in sizes.Children())
{
var sizeName = (string)sz.SelectToken("label");
var productCount = (JArray)sz.SelectToken("products");
if (productCount.Count > 0)
{
details.AddSize(sizeName, "Unknown");
}
}
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
if (document == null)
{
Logger.Instance.WriteErrorLog($"Can't Connect to chmielna website");
throw new WebException("Can't connect to website");
}
var root = document.DocumentNode;
var sizeNodes = root.SelectNodes("//div[@class='selector']/ul/li");
var sizes = sizeNodes?.Select(node => node.GetAttributeValue("data-sizeeu", null) ??
node.GetAttributeValue("data-sizeuk", null) ??
node.GetAttributeValue("data-value", null)).ToList();
var name = root.SelectSingleNode("//div[@class='product__name']/h1")?.InnerText.Trim();
var priceNode = root.SelectSingleNode("//span[@class='product__price_shop']");
var price = Utils.ParsePrice(priceNode?.InnerText);
var image = root.SelectSingleNode("//div[@class='item zoom']/img")?.GetAttributeValue("src", null);
ProductDetails result = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
Debug.Assert(sizes != null, nameof(sizes) + " != null");
foreach (var size in sizes)
{
result.AddSize(size, "Unknown");
}
return(result);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
var asd = document.InnerHtml;
const string xPath = "//select[@id='input-option11842']/option";
var nodes = document.SelectNodes(xPath);
if (nodes == null)
{
throw new Exception();
}
var sizes = nodes.Select(node => node.InnerText.Trim()).Where(element => !element.Contains("Seçiniz"));
ProductDetails details = new ProductDetails();
foreach (var size in sizes)
{
details.AddSize(size, "Unknown");
}
return(details);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
if (document == null)
{
Logger.Instance.WriteErrorLog($"Can't Connect to uptherestore website");
throw new WebException("Can't connect to website");
}
var root = document.DocumentNode;
var sizeNodes = root.SelectSingleNode("//ul[@id = 'configurable_swatch_size']")
.SelectNodes(".//li/a/span[not(@class = 'x')]");
var sizes = sizeNodes?.Select(node => node?.InnerText).ToList();
var name = root.SelectSingleNode("//h1[@itemprop='name']")?.InnerText.Replace("<br>", "\n").Trim();
var priceNode = root.SelectSingleNode(".//span[@class='price'][last()]");
var price = Utils.ParsePrice(priceNode?.InnerText);
var image = root.SelectSingleNode("//div[@class='owl-carousel']//img")?.GetAttributeValue("src", null);
ProductDetails result = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
Debug.Assert(sizes != null, nameof(sizes) + " != null");
foreach (var size in sizes)
{
result.AddSize(size.Trim(), "Unknown");
}
return(result);
}
public override ProductDetails GetProductDetails(string productUrl, CancellationToken token)
{
var document = GetWebpage(productUrl, token);
if (document == null)
{
Logger.Instance.WriteErrorLog($"Can't Connect to woodwood website");
throw new WebException("Can't connect to website");
}
var root = document.DocumentNode;
var sizeNodes = root.SelectNodes("//select[contains(@id, 'form-size')]//option[not(@value='')]");
var sizes = GetSizes(root.InnerHtml);
var name = root.SelectSingleNode("//h1[@class='headline']")?.InnerText.Trim();
var priceNode = root.SelectSingleNode("//span[@class='price']");
var price = Utils.ParsePrice(priceNode?.InnerText);
var image = root.SelectSingleNode("//a[@id='commodity-show-image']/img")?.GetAttributeValue("src", null);
ProductDetails result = new ProductDetails()
{
Price = price.Value,
Name = name,
Currency = price.Currency,
ImageUrl = image,
Url = productUrl,
Id = productUrl,
ScrapedBy = this
};
Debug.Assert(sizes != null, nameof(sizes) + " != null");
foreach (var size in sizes)
{
result.AddSize(size, "Unknown");
}
return(result);
}
