public List<ProductCode> ProcessElkjopProductPage(string url, BackgroundWorker bw, int category) { try { if (bw != null) if (bw.CancellationPending) return new List<ProductCode> { }; string data = FetchPage(url); //string data = DownloadDocument(url); if (data == null || data.Length < 8000) { Log.n("Feil oppstod under nedlasting av side: " + url + " Størrelse: " + data.Length, Color.Red); return new List<ProductCode> { }; } Log.d("Nedlasting fullført (" + url + ") Størrelse: " + data.Length); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(data); HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes("//div[@class='mini-product']"); if (nodes == null || nodes.Count == 0) { Log.n("Fant ingen produkter på side: " + url, Color.Red); return new List<ProductCode> { }; } List<ProductCode> productCodes = new List<ProductCode>(); foreach (HtmlNode node in nodes) { HtmlNode codeNode = node.SelectSingleNode("div[1]/div[1]/small"); if (codeNode == null || String.IsNullOrEmpty(codeNode.InnerText)) { Log.n("Klarte ikke lese en varekode fra produktsiden. Er ignorert", Color.Red); continue; } ProductCode productCode = new ProductCode(); productCode.productCode = codeNode.InnerText; productCode.productCategory = category; HtmlNode stockNode = node.SelectSingleNode("div[1]/div[2]/div[2]/span[2]"); if (stockNode != null && !String.IsNullOrEmpty(stockNode.InnerText) && stockNode.InnerText.Contains("På nettlager")) productCode.productInternetStock = ParseElkjopStock(stockNode.InnerText); HtmlNode priceNode = node.SelectSingleNode("div[1]/div[3]/span/span"); if (priceNode != null && !String.IsNullOrEmpty(priceNode.InnerText)) productCode.productInternetPrize = ParseElkjopPrize(priceNode.InnerText); productCodes.Add(productCode); } if (productCodes.Count == 0) Log.n("Fant ingen produkter på side: " + url, Color.Red); return productCodes; } catch (Exception ex) { Log.Unhandled(ex); } return new List<ProductCode> { }; }
private List<PrisguideProduct> ProcessProductUrls(List<string> productUrls, BackgroundWorker bw) { List<PrisguideProduct> prisguideProducts = new List<PrisguideProduct> { }; int pos = 0; int count = productUrls.Count; foreach (string url in productUrls) { if (bw != null) { if (bw.CancellationPending) return prisguideProducts; bw.ReportProgress(pos, new StatusProgress(count, null, 5, 100)); main.processing.SetText = "Henter data: " + url; } pos++; PrisguideProduct product = new PrisguideProduct(pos); product.productUrl = url; try { product.prisguideId = ParsePrisguideIdFromUrl(url); Log.d("Prisguide #" + pos + ": Laster ned side: " + url); string data = FetchPage(url); //string data = DownloadDocument(url); if (data == null || data.Length < 2000) { Log.d("Prisguide #" + pos + ": Feil med nedlasting av produktside"); product.status = PrisguideProduct.STATUS_DL_ERROR; prisguideProducts.Add(product); continue; } HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(data); HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes("//div[@class='productline']/div[@class='price-info']"); if (nodes == null || nodes.Count == 0) { Log.d("Prisguide #" + pos + ": Fant ingen priser på produktsiden"); product.status = PrisguideProduct.STATUS_NOT_FOUND; prisguideProducts.Add(product); continue; } string prevCode = ""; foreach (HtmlNode elementNode in nodes) { if (bw != null) if (bw.CancellationPending) return prisguideProducts; HtmlNode prizeNode = elementNode.SelectSingleNode("div[@class='price']/a"); if (prizeNode == null) continue; HtmlAttribute productUrl = prizeNode.Attributes["href"]; if (productUrl == null || productUrl.Value == null || !productUrl.Value.Contains("elkjop.no")) continue; try { int indexStart = productUrl.Value.IndexOf("&spid=") + 6; int indexEnd = productUrl.Value.IndexOf("&", indexStart); string code = productUrl.Value.Substring(indexStart, indexEnd - indexStart); if (!prevCode.Equals(code) && !String.IsNullOrEmpty(code)) { ProductCode productCode = new ProductCode(); productCode.productCode = code; if (String.IsNullOrEmpty(prizeNode.InnerText)) Log.d("Prisguide #" + pos + ": Fant ikke pris på varekoden " + code); else productCode.productInternetPrize = ParseElkjopPrize(prizeNode.InnerText); HtmlNode stockNode = elementNode.SelectSingleNode("div[@class='stock stock-green']"); if (stockNode != null && stockNode.InnerText != null && stockNode.InnerText.Contains("på lager")) productCode.productInternetStock = ParseElkjopStock(stockNode.InnerText); product.productCodes.Add(productCode); prevCode = code; } else Log.d("Prisguide #" + pos + ": Vi har allerede denne varekoden: " + code); } catch (Exception) { Log.d("Prisguide #" + pos + ": Feil oppstod under produkt søk på produktsiden: " + url + " - Prøver neste.."); } } if (product.productCodes.Count > 0) product.status = PrisguideProduct.STATUS_OK; else product.status = PrisguideProduct.STATUS_NOT_FOUND; } catch (Exception ex) { Log.Unhandled(ex); product.status = PrisguideProduct.STATUS_EXCEPTION; } prisguideProducts.Add(product); } return prisguideProducts; }