private void ScrapBrowserLoadCompleted(HttpResponseMessage result, int?costmin) { var results = new List <Advertisement>(); if (!result.IsSuccessStatusCode) { _dispatcher.Invoke(() => { AsyncScrapCompleted?.Invoke(this, results, false, null); }); return; } var response = result.Content.ReadAsStringAsync().Result; return; }
private void ScrapBrowserLoadCompleted(string apiResponse, int?costMin) { try { var results = new List <Advertisement>(); var parsedResponse = JsonConvert.DeserializeObject <BvaSearchResponse>(apiResponse); if (parsedResponse == null || !parsedResponse.lots.Any()) { _dispatcher.Invoke(() => { AsyncScrapCompleted?.Invoke(this, results, false, null); }); return; } foreach (var add in parsedResponse.lots) { results.Add(new Advertisement { Name = add.title, Description = add.description + Environment.NewLine + add.startDate + Environment.NewLine + add.endDate, Price = (add.latestBidAmount ?? 0) + add.currencyCode, ImageUrl = add.thumbnailUrl, PageUrl = add.lotPageUrl, }); } _dispatcher.Invoke(() => { AsyncScrapCompleted?.Invoke(this, results, false, null); }); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); _dispatcher.Invoke(() => { AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); }); } }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete) { return; } var results = new List <Advertisement>(); IEnumerable <HtmlElement> ads = null; try { ads = _scrapBrowser.Document.ElementsByClass("div", "row").ToList()[8].ElementsByClass("div", "col-md-6"); if (!ads.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); return; } try { foreach (var add in ads) { var label = add.ElementsByClass("div", "img-wrapper")?.FirstOrDefault()?.InnerText; if (label == null) { continue; } if (label != null) { if (label.Contains("Sold")) { continue; } } var wrapper = add.ElementsByClass("div", "wrapper")?.FirstOrDefault(); var price = wrapper.ElementsByClass("div", "price")?.FirstOrDefault()?.InnerText; if (price != "POA") { price = price.Replace(" ", "").Replace(",", "").Replace("€", "").Replace(".", "").Replace("-", ""); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { price = decimalPrice.ToString("### ###") + " €"; if (decimalPrice < _lastCostMin) { continue; } if (decimalPrice > _lastCostMax) { continue; } } } var description = wrapper.ElementsByClass("div", "description")?.FirstOrDefault()?.InnerText; description = Regex.Replace(description, @"\s+", ""); var title = wrapper.ElementsByClass("div", "title")?.FirstOrDefault()?.InnerHtml; title = Regex.Replace(title, @"\s+", ""); var href = SafeExtractHref(add); var src = SafeExtractSrc(add); if (src == null || src == " " || src == "") { src = "Images/noimage.png"; } results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = src, PageUrl = href, }); } bool hasMorePages = false; _scrapDone = true; AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } finally { if (_scrapBrowser != null) { _scrapBrowser.Dispose(); _scrapBrowser = null; } } }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { try { string[] filters = null; string[] searchTextWords = _searchText.ToLower().Split(' '); string apiResponse = _scrapBrowser.Document.Body.InnerHtml; var start = apiResponse.IndexOf("{"); var end = apiResponse.LastIndexOf("}</SCRIPT>"); apiResponse = apiResponse.Substring(start, end - start + 1).Trim(); if (!string.IsNullOrEmpty(_synonyms)) { filters = _synonyms.ToLower().Split(';'); } var results = new List <Advertisement>(); var parsedResponse = JsonConvert.DeserializeObject <MarktplaatsSearchResponse>(apiResponse); if (parsedResponse == null || !parsedResponse.props.pageProps.searchRequestAndResponse.listings.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); } foreach (var add in parsedResponse.props.pageProps.searchRequestAndResponse.listings) { string img; string price; string title; bool continueFlag = false; bool breakFlag = false; if (add.imageUrls != null && add.imageUrls.Any()) { img = "https:" + add.imageUrls[0]; } else { img = "Images/noimage.png"; } title = add?.title; if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower())) { string testTitle = title.ToLower().Replace(" ", ""); foreach (var word in searchTextWords) { if (!testTitle.Contains(word)) { // Checking if title contains filters if (filters != null && filters.Length != 0) { foreach (var filter in filters) { if (testTitle.Contains(filter) && !filter.Equals("")) { breakFlag = true; } } } if (breakFlag) { break; } continueFlag = true; break; } } } if (continueFlag) { continue; } if (add.priceInfo.priceType.ToUpper() != "FIXED") { price = "POA"; } else { price = (add.priceInfo.priceCents / 100).ToString(); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { price = decimalPrice.ToString("### ###") + " €"; if (_lastCostMin.HasValue && decimalPrice < _lastCostMin) { continue; } if (_lastCostMax.HasValue && decimalPrice > _lastCostMax) { continue; } } } results.Add(new Advertisement { Name = title, Description = add.description + Environment.NewLine + add.sellerInformation.sellerName, Price = price, ImageUrl = img, PageUrl = "https://www.marktplaats.nl" + add?.vipUrl, }); } bool hasNextPage; decimal resultCount = parsedResponse.props.pageProps.searchRequestAndResponse.totalResultCount; decimal totalPages = Math.Ceiling(resultCount / 30); if (parsedResponse.props.pageProps.page + 1 < totalPages) { hasNextPage = true; } else { hasNextPage = false; } AsyncScrapCompleted?.Invoke(this, results, hasNextPage, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete) { return; } IEnumerable <HtmlElement> ads = null; string[] filters = null; string[] searchTextWords = _searchText.ToLower().Split(' '); if (!string.IsNullOrEmpty(_synonyms)) { filters = _synonyms.ToLower().Split(';'); } var results = new List <Advertisement>(); try { ads = _scrapBrowser.Document.ElementsByClass("div", "styled__Wrapper-sc-1kpvi4z-0"); if (!ads.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); return; } try { foreach (var add in ads) { var title = add.ElementsByClass("div", "styled__SubjectWrapper-sc-1kpvi4z-12")?.FirstOrDefault().InnerText; bool continueFlag = false; bool breakFlag = false; //Checking if title contains each request text word if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower())) { string testTitle = title.ToLower().Replace(" ", ""); foreach (var word in searchTextWords) { if (!testTitle.Contains(word)) { // Checking if title contains filters if (filters != null && filters.Length != 0) { foreach (var filter in filters) { if (testTitle.Contains(filter) && !filter.Equals("")) { breakFlag = true; } } } if (breakFlag) { break; } continueFlag = true; break; } } } if (continueFlag) { continue; } var price = add.ElementsByClass("div", "styled__SalesInfo-sc-1kpvi4z-22")?.FirstOrDefault()?.InnerText; if (price != null && price.ToLower().Contains("ex. moms")) { price = price.Remove(price.IndexOf("(")); } if (price == null || !Regex.IsMatch(price, @"^[0-9]")) { price = "POA"; } else { price = price.Replace(" ", "").Replace(",", "").Replace("kr", ""); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { var euroPrice = (int)(Math.Round(decimalPrice * _currentRate)); price = euroPrice.ToString("### ###") + " €"; if (euroPrice < _lastCostMin) { continue; } if (euroPrice > _lastCostMax) { continue; } } } var description = _scrapBrowser.Document.ElementsByClass("div", "styled__ParamsWrapper-sc-1kpvi4z-15")?.FirstOrDefault()?.InnerText; var src = SafeExtractSrc(add.ElementsByClass("div", "ListImage__Wrapper-sc-1rp77jc-0").FirstOrDefault()); if (src == null || src == " " || src == "") { src = "Images/noimage.png"; } var href = SafeExtractHref(add); results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = src, PageUrl = href, }); } var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document); _scrapDone = true; AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } /*finally * { * if (_scrapBrowser != null) * { * _scrapBrowser.Dispose(); * _scrapBrowser = null; * } * }*/ }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete) { return; } IEnumerable <HtmlElement> ads = null; var results = new List <Advertisement>(); try { ads = _scrapBrowser.Document.ElementsByClass("div", "lot__wrapper"); if (!ads.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); return; } try { foreach (var add in ads) { var divDes = add.ElementsByClass("div", "lot__footer")?.FirstOrDefault(); if (divDes.InnerText.ToLower().Contains("lot is closed")) { continue; } var title = add.ElementsByClass("div", "lot__title")?.FirstOrDefault()?.InnerText; var description = divDes?.InnerText; var div = add.ElementsByClass("div", "lot__currentbid")?.FirstOrDefault(); var price = div.ElementsByClass("span", "lot__bidamount")?.FirstOrDefault()?.InnerText; var a = add.ElementsByClass("a", "lot__image")?.FirstOrDefault(); var src = SafeExtractSrc(a); if (src == null || src == " " || src == "") { src = "Images/noimage.png"; } var href = SafeExtractHref(add); results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = src, PageUrl = href, }); } _scrapDone = true; AsyncScrapCompleted?.Invoke(this, results, false, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } finally { if (_scrapBrowser != null) { _scrapBrowser.Dispose(); _scrapBrowser = null; } } }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete) { return; } IEnumerable <HtmlElement> ads = null; string[] filters = null; string[] searchTextWords = _searchText.ToLower().Split(' '); if (!string.IsNullOrEmpty(_synonyms)) { filters = _synonyms.ToLower().Split(';'); } var results = new List <Advertisement>(); try { ads = _scrapBrowser.Document.ElementsByClass("div", "row-listing"); if (!ads.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); return; } try { foreach (var add in ads) { var divTit = add.ElementsByClass("div", "title")?.FirstOrDefault(); if (divTit == null) { continue; } var title = divTit?.InnerText; bool continueFlag = false; bool breakFlag = false; //Checking if title contains each request text word if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower())) { string testTitle = title.ToLower().Replace(" ", ""); foreach (var word in searchTextWords) { if (!testTitle.Contains(word)) { // Checking if title contains filters if (filters != null && filters.Length != 0) { foreach (var filter in filters) { if (testTitle.Contains(filter) && !filter.Equals("")) { breakFlag = true; } } } if (breakFlag) { break; } continueFlag = true; break; } } } if (continueFlag) { continue; } var description = add?.ElementsByClass("div", "listing-infos")?.FirstOrDefault()?.InnerText; var price = add?.ElementsByClass("div", "listing-price")?.FirstOrDefault()?.InnerText; if (price == null) { price = "POA"; } else { price = price.Replace(",", "").Replace("€", ""); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { price = decimalPrice.ToString("### ###") + " €"; if (decimalPrice < _lastCostMin) { continue; } if (decimalPrice > _lastCostMax) { continue; } } } var divImg = add.ElementsByClass("div", "picture")?.FirstOrDefault(); var src = SafeExtratImgSrc(divImg); if (src == null || src == " " || src == "") { src = "Images/noimage.png"; } var href = SafeExtractHref(add); results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = src, PageUrl = href, }); } var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document); _scrapDone = true; AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete) { return; } HtmlElementCollection adsli = null; var results = new List <Advertisement>(); try { var ul = _scrapBrowser.Document?.ElementsByClass("ul", "lot-list").FirstOrDefault(); if (ul == null) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } adsli = ul.GetElementsByTagName("li"); if (adsli.Count == 0) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); return; } try { foreach (HtmlElement add in adsli) { var title = add.ElementsByClass("a", "title")?.FirstOrDefault()?.InnerText; if (title == null) { continue; } var description = add.ElementsByClass("div", "lot-description")?.FirstOrDefault()?.InnerText; var price = add.ElementsByClass("span", "time-parts")?.FirstOrDefault()?.InnerText; var divSrc = add.ElementsByClass("div", "thumbnail")?.FirstOrDefault(); var src = SafeExtractSrc(divSrc); if (src == null || src == " " || src == "") { src = "Images/noimage.png"; } var href = SafeExtractHref(divSrc); results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = src, PageUrl = "https://www.troostwijkauctions.com" + href, }); } _scrapDone = true; AsyncScrapCompleted?.Invoke(this, results, false, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } finally { if (_scrapBrowser != null) { _scrapBrowser.Dispose(); _scrapBrowser = null; } } }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete) { return; } IEnumerable <HtmlElement> ads = null; var results = new List <Advertisement>(); try { ads = _scrapBrowser.Document.ElementsByClass("div", "listingBox").ToList(); if (!ads.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); return; } try { foreach (var add in ads) { var title = add.ElementsByClass("span", "objectMaker")?.FirstOrDefault()?.InnerText + add.ElementsByClass("span", "objectTitle")?.FirstOrDefault()?.InnerText; if (title == "" || title == null) { continue; } var description = add.ElementsByClass("span", "end")?.FirstOrDefault()?.InnerText; var price = add.ElementsByClass("div", "highestBid")?.FirstOrDefault()?.InnerText; if (price == null) { price = "POA"; } else { price = price.Replace(" ", "").Replace(",", "").Replace("kr", ""); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { var euroPrice = (int)(Math.Round(decimalPrice * _currentRate)); price = euroPrice.ToString("### ###") + " €"; } } var styles = SafeExtractStyles(add); styles = styles?.Remove(0, styles.IndexOf("(") + 1); var src = styles?.Remove(styles.IndexOf(")")); if (src == null || src == " " || src == "") { src = "Images/noimage.png"; } var href = SafeExtractHref(add); results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = "https:" + src, PageUrl = href, }); } _scrapDone = true; AsyncScrapCompleted?.Invoke(this, results, false, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } finally { if (_scrapBrowser != null) { _scrapBrowser.Dispose(); _scrapBrowser = null; } } }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete) { return; } var results = new List <Advertisement>(); string[] filters = null; string[] searchTextWords = _searchText.ToLower().Split(' '); if (!string.IsNullOrEmpty(_synonyms)) { filters = _synonyms.ToLower().Split(';'); } IEnumerable <HtmlElement> cardsList = null; try { cardsList = _scrapBrowser.Document.ElementsByClass("div", "advertisement-list-item"); if (cardsList == null && !cardsList.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); return; } try { foreach (var cardelement in cardsList) { var title = cardelement.ElementsByClass("a", "card-title").FirstOrDefault().InnerText; bool continueFlag = false; bool breakFlag = false; //Checking if title contains each request text word if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower())) { string testTitle = title.ToLower().Replace(" ", ""); foreach (var word in searchTextWords) { if (!testTitle.Contains(word)) { // Checking if title contains filters if (filters != null && filters.Length != 0) { foreach (var filter in filters) { if (testTitle.Contains(filter) && !filter.Equals("")) { breakFlag = true; } } } if (breakFlag) { break; } continueFlag = true; break; } } } if (continueFlag) { continue; } var price = cardelement.ElementsByClass("p", "card-text")?.FirstOrDefault()?.InnerText; int trashIndex = price.LastIndexOf('\n'); price = price.Substring(trashIndex + 1); if (!string.IsNullOrWhiteSpace(price)) { if (!Regex.IsMatch(price, @"^€[0-9]")) { price = "PAO"; } else { price = price.Replace("Prijs: ", "").Replace(".", "").Replace(",", "").Replace("€", "").Replace("-", "").Replace(" ", ""); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { price = decimalPrice.ToString("### ###") + " €"; if (decimalPrice < _lastCostMin) { continue; } if (decimalPrice > _lastCostMax) { continue; } } } } else { price = "PAO"; } var description = cardelement.ElementsByClass("ul", "list-properties")?.FirstOrDefault()?.InnerText; var src = SafeExtractSrc(cardelement); if (src == null || src == " " || src == "") { src = "Images/noimage.png"; } var href = SafeExtractHref(cardelement); results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = src, PageUrl = href, }); } var pagination = _scrapBrowser.Document.ElementsByClass("ul", "pagination").FirstOrDefault(); var hasMorePages = ScrapHasMorePages(pagination); _scrapDone = true; AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone || _scrapBrowser.ReadyState == WebBrowserReadyState.Loading || _scrapBrowser.ReadyState == WebBrowserReadyState.Loaded || _scrapBrowser.ReadyState == WebBrowserReadyState.Uninitialized) { return; } IEnumerable <HtmlElement> ads = null; string[] filters = null; string[] searchTextWords = _searchText.ToLower().Split(' '); if (!string.IsNullOrEmpty(_synonyms)) { filters = _synonyms.ToLower().Split(';'); } var results = new List <Advertisement>(); try { ads = _scrapBrowser.Document.ElementsByClass("li", "mb-20"); if (!ads.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); return; } try { foreach (HtmlElement add in ads) { var title = add.ElementsByClass("h3", "text-18px")?.FirstOrDefault()?.InnerText; bool continueFlag = false; bool breakFlag = false; if (title == null) { continue; } //Checking if title contains each request text word if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower())) { string testTitle = title.ToLower().Replace(" ", ""); foreach (var word in searchTextWords) { if (!testTitle.Contains(word)) { // Checking if title contains filters if (filters != null && filters.Length != 0) { foreach (var filter in filters) { if (testTitle.Contains(filter) && !filter.Equals("")) { breakFlag = true; } } } if (breakFlag) { break; } continueFlag = true; break; } } } if (continueFlag) { continue; } var price = add.ElementsByClass("p", "price")?.FirstOrDefault()?.InnerText; if (!string.IsNullOrWhiteSpace(price)) { if (price.ToLower().Contains("ring for pris")) { price = "POA"; } else { price = price.Replace("Pris:", "").Replace("DKK", "").Replace("-", "").Replace(".", "").Replace(",", "").Replace("€", ""); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { var euroPrice = (int)(Math.Round(decimalPrice * _currentRate)); price = euroPrice.ToString("### ###") + " €"; if (euroPrice < _lastCostMin) { continue; } if (euroPrice > _lastCostMax) { continue; } } } } var href = SafeExtractHref(add); var description = add.ElementsByClass("p", "description")?.FirstOrDefault()?.InnerText + Environment.NewLine + add.ElementsByClass("p", "location-info")?.FirstOrDefault()?.InnerText; var moreDescription = add.ElementsByClass("li", "year")?.FirstOrDefault()?.InnerText; var moremoreDescription = add.ElementsByClass("li", "hours")?.FirstOrDefault()?.InnerText; var moremoremoreDescription = add.ElementsByClass("li", "hp")?.FirstOrDefault()?.InnerText; description = "Year: " + moreDescription + " Hours: " + moremoreDescription + " HP: " + moremoremoreDescription + Environment.NewLine + description; var srcDiv = add.ElementsByClass("figure", "image")?.FirstOrDefault(); var imgSrc = SafeExtractSrc(srcDiv); if (imgSrc == null || imgSrc == " " || imgSrc == "") { imgSrc = "Images/noimage.png"; } results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = imgSrc, PageUrl = href, }); } var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document); _scrapDone = true; _scrapBrowser.Stop(); AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } /*finally * { * if (_scrapBrowser != null) * { * _scrapBrowser.Dispose(); * _scrapBrowser = null; * } * }*/ }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete) { return; } IEnumerable <HtmlElement> ads = null; string[] filters = null; string[] searchTextWords = _searchText.ToLower().Split(' '); if (!string.IsNullOrEmpty(_synonyms)) { filters = _synonyms.ToLower().Split(';'); } var results = new List <Advertisement>(); try { ads = _scrapBrowser.Document.ElementsByClass("div", "mp-Listing-card-flex"); if (!ads.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); return; } try { foreach (var add in ads) { var descriptionDiv = add.ElementsByClass("div", "mp-Listing-card-flex-content--align-bottom")?.FirstOrDefault(); var price = descriptionDiv?.ElementsByClass("div", "mp-Listing-card-flex-price")?.FirstOrDefault()?.InnerText; if (price == null) { continue; } var title = add.ElementsByClass("div", "mp-Listing-card-flex-title")?.FirstOrDefault()?.InnerText; bool continueFlag = false; bool breakFlag = false; //Checking if title contains each request text word if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower())) { string testTitle = title.ToLower().Replace(" ", ""); foreach (var word in searchTextWords) { if (!testTitle.Contains(word)) { // Checking if title contains filters if (filters != null && filters.Length != 0) { foreach (var filter in filters) { if (testTitle.Contains(filter) && !filter.Equals("")) { breakFlag = true; } } } if (breakFlag) { break; } continueFlag = true; break; } } } if (continueFlag) { continue; } if (!string.IsNullOrWhiteSpace(price)) { if (price.ToUpper().Contains("OP AANVRAAG")) { price = "On Request"; } else if (price.ToUpper().Contains("BIEDEN")) { price = "Offer"; } else if (price.ToUpper().Contains("N.O.T.K.")) { price = "To be agreed"; } else if (price.ToUpper().Contains("ZIE OMSCHRIJVING")) { price = "See Description"; } else if (price.ToUpper().Contains("GERESERVEERD")) { price = "Reserved"; } else { price = price.Replace(",", "").Replace(".", "").Replace("€", ""); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { decimalPrice /= 100; price = decimalPrice.ToString("### ###") + " €"; if (decimalPrice < _lastCostMin) { continue; } if (decimalPrice > _lastCostMax) { continue; } } } } var description = descriptionDiv.ElementsByClass("div", "mp-Listing-card-flex-location-block")?.FirstOrDefault()?.InnerText; var srcDiv = add.ElementsByClass("div", "mp-Listing-card-flex-image-block")?.FirstOrDefault(); var src = SafeExtractSrc(srcDiv); if (src == null || src == " " || src == "") { src = "Images/noimage.png"; } var href = SafeExtractHref(add); results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = src, PageUrl = href, }); } var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document); _scrapDone = true; AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } /*finally * { * if (_scrapBrowser != null) * { * _scrapBrowser.Dispose(); * _scrapBrowser = null; * } * }*/ }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { try { string[] filters = null; string[] searchTextWords = _searchText.ToLower().Split(' '); string apiResponse = _scrapBrowser.Document.Body.InnerHtml; if (apiResponse.IndexOf("json\">{") != -1) { apiResponse = apiResponse.Substring(apiResponse.IndexOf("json\">{")); } var start = apiResponse.IndexOf("{"); var end = apiResponse.LastIndexOf("}</script><script"); apiResponse = apiResponse.Substring(start, end - start + 1).Trim(); if (!string.IsNullOrEmpty(_synonyms)) { filters = _synonyms.ToLower().Split(';'); } var results = new List <Advertisement>(); var parsedResponse = JsonConvert.DeserializeObject <TradeMachinesSearchResponse>(apiResponse); RequestCounter++; if (RequestCounter > 1) { return; } if (parsedResponse == null || !parsedResponse.props.pageProps.resultsState.rawResults[0].hits.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } foreach (var add in parsedResponse.props.pageProps.resultsState.rawResults[0].hits) { string img; string price; string title; bool continueFlag = false; bool breakFlag = false; if (add.product.isSparePart == true) { continue; } if (add.hasImg) { img = "https://dizv3061bgivy.cloudfront.net" + add.imgId; } else { img = "Images/noimage.png"; } if (add.fr.title != null && add.product.name != null) { if (add.fr.title.Length >= add.product.name.Length) { title = add.fr.title; } else { title = add.product.name; } } else { if (add.fr.title == null) { title = add.product.name; } else { title = add.fr.title; } } if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower())) { string testTitle = title.ToLower().Replace(" ", ""); foreach (var word in searchTextWords) { if (!testTitle.Contains(word)) { // Checking if title contains filters if (filters != null && filters.Length != 0) { foreach (var filter in filters) { if (testTitle.Contains(filter) && !filter.Equals("")) { breakFlag = true; } } } if (breakFlag) { break; } continueFlag = true; break; } } } if (continueFlag) { continue; } if (!add.hasPrice) { price = "POA"; } else { price = Math.Round(add.price).ToString(); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { price = decimalPrice.ToString("### ###") + " €"; if (_lastCostMin.HasValue && decimalPrice < _lastCostMin) { continue; } if (_lastCostMax.HasValue && decimalPrice > _lastCostMax) { continue; } } } results.Add(new Advertisement { Name = title, Description = "Year: " + add?.year + Environment.NewLine + add.location?.country + " " + add.location?.state + " " + add.location?.city + Environment.NewLine + add.seller?.name, Price = price, ImageUrl = img, PageUrl = "https://trademachines.fr/lots/" + add?.objectID, }); } var parsedResponseRawResult = parsedResponse.props.pageProps.resultsState.rawResults[0]; bool hasNextPage; if (parsedResponseRawResult.nbPages > parsedResponseRawResult.page + 1) { hasNextPage = true; } else { hasNextPage = false; } AsyncScrapCompleted?.Invoke(this, results, hasNextPage, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone) { return; } IEnumerable <HtmlElement> ads = null; string[] filters = null; string[] searchTextWords = _searchText.ToLower().Split(' '); if (!string.IsNullOrEmpty(_synonyms)) { filters = _synonyms.ToLower().Split(';'); } var results = new List <Advertisement>(); try { ads = _scrapBrowser.Document.ElementsByClass("div", "machine_list_entry"); if (!ads.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); return; } try { foreach (var add in ads) { var title = add.ElementsByClass("span", "machine_entry_title")?.FirstOrDefault()?.InnerText; bool continueFlag = false; bool breakFlag = false; //Checking if title contains each request text word if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower())) { string testTitle = title.ToLower().Replace(" ", ""); foreach (var word in searchTextWords) { if (!testTitle.Contains(word)) { // Checking if title contains filters if (filters != null && filters.Length != 0) { foreach (var filter in filters) { if (testTitle.Contains(filter) && !filter.Equals("")) { breakFlag = true; } } } if (breakFlag) { break; } continueFlag = true; break; } } } if (continueFlag) { continue; } var price = add.ElementsByClass("span", "machine_entry_price_price")?.FirstOrDefault()?.InnerText; if (price == null) { price = add.ElementsByClass("span", "machine_entry_price_price specialOrange")?.FirstOrDefault()?.InnerText; } if (!string.IsNullOrWhiteSpace(price)) { if (price.ToLower().Contains("preis auf anfrage")) { price = "POA"; } if (price.ToLower().Contains("auktion")) { price = "Auction"; } else { price = price.Replace(".", "").Replace(",", "").Replace("€", ""); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { price = decimalPrice.ToString("### ###") + " €"; if (decimalPrice < _lastCostMin) { continue; } if (decimalPrice > _lastCostMax) { continue; } } } } var description = add.ElementsByClass("span", "machine_entry_description")?.FirstOrDefault()?.InnerText; var div = add.ElementsByClass("div", "machine_entry_image")?.FirstOrDefault(); var src = SafeExtractSrc(div); if (src != null && src[0] != 'h') { src = "https://www.traktorpool.de" + src; } else if (src == null || src == " " || src == "") { src = "Images/noimage.png"; } var href = SafeExtractHref(add); results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = src, PageUrl = href, }); } var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document); _scrapDone = true; AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } /*finally * { * if (_scrapBrowser != null) * { * _scrapBrowser.Dispose(); * _scrapBrowser = null; * } * }*/ }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete) { return; } IEnumerable <HtmlElement> ads = null; var results = new List <Advertisement>(); try { ads = _scrapBrowser.Document.ElementsByClass("div", "_3DFQ-"); if (!ads.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); return; } try { foreach (var add in ads) { var price = add.ElementsByClass("div", "_2OJ8g")?.FirstOrDefault()?.InnerText; if (price == null) { price = "POA"; } else { price = price.Replace(",", "").Replace("€", "").Replace(".", ""); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { price = decimalPrice.ToString("### ###") + " €"; if (_lastCostMin.HasValue && decimalPrice < _lastCostMin) { continue; } if (_lastCostMax.HasValue && decimalPrice > _lastCostMax) { continue; } } } var title = add.ElementsByClass("div", "_2tubl")?.FirstOrDefault()?.InnerText; var description = add.ElementsByClass("div", "_32V5I").FirstOrDefault()?.InnerText; var divImg = add.ElementsByClass("div", "LazyLoad is-visible")?.FirstOrDefault(); var src = SafeExtractSrc(divImg); if (src == null || src == " " || src == "") { src = "Images/noimage.png"; } var href = SafeExtractHref(add); results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = src, PageUrl = href, }); } var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document); _scrapDone = true; AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } finally { if (_scrapBrowser != null) { _scrapBrowser.Dispose(); _scrapBrowser = null; } } }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete) { return; } //while (!_rateLoaded) { //System.Threading.Thread.Sleep(500); } var results = new List <Advertisement>(); try { var table = _scrapBrowser.Document.ElementsByClass("table", "productList").FirstOrDefault(); var ads = table.GetElementsByTagName("tr"); if (ads.Count == 0) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } foreach (HtmlElement add in ads) { if (add == ads[0]) { continue; } var price = add.ElementsByClass("TD", "col_Price")?.FirstOrDefault()?.InnerText; if (price.ToUpper().Contains("CALL US")) { price = "POA"; } else { price = price.Replace(",", "").Replace("£", "").Replace(" ", ""); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { var euroPrice = (int)(Math.Round(decimalPrice * _currentRate)); price = euroPrice.ToString("### ###") + " €"; if (euroPrice < _lastCostMin) { continue; } if (euroPrice > _lastCostMax) { continue; } } } var description = "Year: " + add.ElementsByClass("TD", "col_Year")?.FirstOrDefault()?.InnerText + "\n" + "Hours: " + add.ElementsByClass("TD", "col_Hours")?.FirstOrDefault()?.InnerText + "\n" + "Width: " + add.ElementsByClass("TD", "col_Width")?.FirstOrDefault()?.InnerText; var hreftd = add.ElementsByClass("TD", "viewMoreButton")?.FirstOrDefault(); var href = SafeExtractHref(hreftd); if (href != null && href[0] != 'h') { href = "https://www.farol.co.uk" + href; } var a = add.ElementsByClass("TD", "col_Image")?.FirstOrDefault(); var src = SafeExtractSrc(a); if (src != null && src[0] != 'h') { src = "https://www.farol.co.uk" + src; } var title = add.ElementsByClass("TD", "col_Make")?.FirstOrDefault()?.InnerText + " " + add.ElementsByClass("TD", "col_Model")?.FirstOrDefault()?.InnerText; results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = src, PageUrl = href, }); } var hasMorePages = false; _scrapDone = true; AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } }
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete) { return; } IEnumerable <HtmlElement> ads = null; string[] filters = null; string[] searchTextWords = _searchText.ToLower().Split(' '); if (!string.IsNullOrEmpty(_synonyms)) { filters = _synonyms.ToLower().Split(';'); } var results = new List <Advertisement>(); try { ads = _scrapBrowser.Document.ElementsByClass("div", "item-list"); if (!ads.Any()) { AsyncScrapCompleted?.Invoke(this, results, false, null); return; } } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); return; } try { foreach (var add in ads) { var info = add.ElementsByClass("div", "add-details")?.FirstOrDefault(); if (info == null) { continue; } var title = SafeExtrctTitle(info); bool continueFlag = false; bool breakFlag = false; //Checking if title contains each request text word if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower())) { string testTitle = title.ToLower().Replace(" ", ""); foreach (var word in searchTextWords) { if (!testTitle.Contains(word)) { // Checking if title contains filters if (filters != null && filters.Length != 0) { foreach (var filter in filters) { if (testTitle.Contains(filter) && !filter.Equals("")) { breakFlag = true; } } } if (breakFlag) { break; } continueFlag = true; break; } } } if (continueFlag) { continue; } var price = add.ElementsByClass("h3", "item-price-small")?.FirstOrDefault()?.InnerText; var priceTax = add.ElementsByClass("h2", "item-price")?.FirstOrDefault()?.InnerText; if (priceTax != null && priceTax.ToUpper() == "PRICE ON APPLICATION") { price = "PAO"; } else { price = price.Replace(",", "").Replace("€", "").Replace(".", "").Replace("Net", ""); if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice)) { price = decimalPrice.ToString("### ###") + " €"; if (_lastCostMin.HasValue && decimalPrice < _lastCostMin) { continue; } if (_lastCostMax.HasValue && decimalPrice > _lastCostMax) { continue; } } } var description = add.ElementsByClass("span", "info-row").FirstOrDefault()?.InnerText; var divImg = add.ElementsByClass("div", "add-image")?.FirstOrDefault(); var src = SafeExtractSrc(divImg); if (src == "") { src = "http://www.pinnacleeducations.in/wp-content/uploads/2017/05/no-image.jpg"; } var href = SafeExtractHref(add); results.Add(new Advertisement { Name = title, Description = description, Price = price, ImageUrl = src, PageUrl = href, }); } var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document); _scrapDone = true; AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null); } catch (Exception ex) { _loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted"); AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception"); } finally { if (_scrapBrowser != null) { _scrapBrowser.Dispose(); _scrapBrowser = null; } } }