private void ScrapBrowserLoadCompleted(HttpResponseMessage result, int?costmin)
{
var results = new List <Advertisement>();
if (!result.IsSuccessStatusCode)
{
_dispatcher.Invoke(() => {
AsyncScrapCompleted?.Invoke(this, results, false, null);
});
return;
}
var response = result.Content.ReadAsStringAsync().Result;
return;
}
private void ScrapBrowserLoadCompleted(string apiResponse, int?costMin)
{
try
{
var results = new List <Advertisement>();
var parsedResponse = JsonConvert.DeserializeObject <BvaSearchResponse>(apiResponse);
if (parsedResponse == null || !parsedResponse.lots.Any())
{
_dispatcher.Invoke(() => {
AsyncScrapCompleted?.Invoke(this, results, false, null);
});
return;
}
foreach (var add in parsedResponse.lots)
{
results.Add(new Advertisement
{
Name = add.title,
Description = add.description + Environment.NewLine + add.startDate + Environment.NewLine + add.endDate,
Price = (add.latestBidAmount ?? 0) + add.currencyCode,
ImageUrl = add.thumbnailUrl,
PageUrl = add.lotPageUrl,
});
}
_dispatcher.Invoke(() => {
AsyncScrapCompleted?.Invoke(this, results, false, null);
});
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
_dispatcher.Invoke(() => {
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
});
}
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete)
{
return;
}
var results = new List <Advertisement>();
IEnumerable <HtmlElement> ads = null;
try
{
ads = _scrapBrowser.Document.ElementsByClass("div", "row").ToList()[8].ElementsByClass("div", "col-md-6");
if (!ads.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
return;
}
try
{
foreach (var add in ads)
{
var label = add.ElementsByClass("div", "img-wrapper")?.FirstOrDefault()?.InnerText;
if (label == null)
{
continue;
}
if (label != null)
{
if (label.Contains("Sold"))
{
continue;
}
}
var wrapper = add.ElementsByClass("div", "wrapper")?.FirstOrDefault();
var price = wrapper.ElementsByClass("div", "price")?.FirstOrDefault()?.InnerText;
if (price != "POA")
{
price = price.Replace(" ", "").Replace(",", "").Replace("€", "").Replace(".", "").Replace("-", "");
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
price = decimalPrice.ToString("### ###") + " €";
if (decimalPrice < _lastCostMin)
{
continue;
}
if (decimalPrice > _lastCostMax)
{
continue;
}
}
}
var description = wrapper.ElementsByClass("div", "description")?.FirstOrDefault()?.InnerText;
description = Regex.Replace(description, @"\s+", "");
var title = wrapper.ElementsByClass("div", "title")?.FirstOrDefault()?.InnerHtml;
title = Regex.Replace(title, @"\s+", "");
var href = SafeExtractHref(add);
var src = SafeExtractSrc(add);
if (src == null || src == " " || src == "")
{
src = "Images/noimage.png";
}
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = src,
PageUrl = href,
});
}
bool hasMorePages = false;
_scrapDone = true;
AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
finally
{
if (_scrapBrowser != null)
{
_scrapBrowser.Dispose();
_scrapBrowser = null;
}
}
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
try
{
string[] filters = null;
string[] searchTextWords = _searchText.ToLower().Split(' ');
string apiResponse = _scrapBrowser.Document.Body.InnerHtml;
var start = apiResponse.IndexOf("{");
var end = apiResponse.LastIndexOf("}</SCRIPT>");
apiResponse = apiResponse.Substring(start, end - start + 1).Trim();
if (!string.IsNullOrEmpty(_synonyms))
{
filters = _synonyms.ToLower().Split(';');
}
var results = new List <Advertisement>();
var parsedResponse = JsonConvert.DeserializeObject <MarktplaatsSearchResponse>(apiResponse);
if (parsedResponse == null || !parsedResponse.props.pageProps.searchRequestAndResponse.listings.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
}
foreach (var add in parsedResponse.props.pageProps.searchRequestAndResponse.listings)
{
string img;
string price;
string title;
bool continueFlag = false;
bool breakFlag = false;
if (add.imageUrls != null && add.imageUrls.Any())
{
img = "https:" + add.imageUrls[0];
}
else
{
img = "Images/noimage.png";
}
title = add?.title;
if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower()))
{
string testTitle = title.ToLower().Replace(" ", "");
foreach (var word in searchTextWords)
{
if (!testTitle.Contains(word))
{
// Checking if title contains filters
if (filters != null && filters.Length != 0)
{
foreach (var filter in filters)
{
if (testTitle.Contains(filter) && !filter.Equals(""))
{
breakFlag = true;
}
}
}
if (breakFlag)
{
break;
}
continueFlag = true;
break;
}
}
}
if (continueFlag)
{
continue;
}
if (add.priceInfo.priceType.ToUpper() != "FIXED")
{
price = "POA";
}
else
{
price = (add.priceInfo.priceCents / 100).ToString();
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
price = decimalPrice.ToString("### ###") + " €";
if (_lastCostMin.HasValue && decimalPrice < _lastCostMin)
{
continue;
}
if (_lastCostMax.HasValue && decimalPrice > _lastCostMax)
{
continue;
}
}
}
results.Add(new Advertisement
{
Name = title,
Description = add.description + Environment.NewLine +
add.sellerInformation.sellerName,
Price = price,
ImageUrl = img,
PageUrl = "https://www.marktplaats.nl" + add?.vipUrl,
});
}
bool hasNextPage;
decimal resultCount = parsedResponse.props.pageProps.searchRequestAndResponse.totalResultCount;
decimal totalPages = Math.Ceiling(resultCount / 30);
if (parsedResponse.props.pageProps.page + 1 < totalPages)
{
hasNextPage = true;
}
else
{
hasNextPage = false;
}
AsyncScrapCompleted?.Invoke(this, results, hasNextPage, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete)
{
return;
}
IEnumerable <HtmlElement> ads = null;
string[] filters = null;
string[] searchTextWords = _searchText.ToLower().Split(' ');
if (!string.IsNullOrEmpty(_synonyms))
{
filters = _synonyms.ToLower().Split(';');
}
var results = new List <Advertisement>();
try
{
ads = _scrapBrowser.Document.ElementsByClass("div", "styled__Wrapper-sc-1kpvi4z-0");
if (!ads.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
return;
}
try
{
foreach (var add in ads)
{
var title = add.ElementsByClass("div", "styled__SubjectWrapper-sc-1kpvi4z-12")?.FirstOrDefault().InnerText;
bool continueFlag = false;
bool breakFlag = false;
//Checking if title contains each request text word
if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower()))
{
string testTitle = title.ToLower().Replace(" ", "");
foreach (var word in searchTextWords)
{
if (!testTitle.Contains(word))
{
// Checking if title contains filters
if (filters != null && filters.Length != 0)
{
foreach (var filter in filters)
{
if (testTitle.Contains(filter) && !filter.Equals(""))
{
breakFlag = true;
}
}
}
if (breakFlag)
{
break;
}
continueFlag = true;
break;
}
}
}
if (continueFlag)
{
continue;
}
var price = add.ElementsByClass("div", "styled__SalesInfo-sc-1kpvi4z-22")?.FirstOrDefault()?.InnerText;
if (price != null && price.ToLower().Contains("ex. moms"))
{
price = price.Remove(price.IndexOf("("));
}
if (price == null || !Regex.IsMatch(price, @"^[0-9]"))
{
price = "POA";
}
else
{
price = price.Replace(" ", "").Replace(",", "").Replace("kr", "");
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
var euroPrice = (int)(Math.Round(decimalPrice * _currentRate));
price = euroPrice.ToString("### ###") + " €";
if (euroPrice < _lastCostMin)
{
continue;
}
if (euroPrice > _lastCostMax)
{
continue;
}
}
}
var description = _scrapBrowser.Document.ElementsByClass("div", "styled__ParamsWrapper-sc-1kpvi4z-15")?.FirstOrDefault()?.InnerText;
var src = SafeExtractSrc(add.ElementsByClass("div", "ListImage__Wrapper-sc-1rp77jc-0").FirstOrDefault());
if (src == null || src == " " || src == "")
{
src = "Images/noimage.png";
}
var href = SafeExtractHref(add);
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = src,
PageUrl = href,
});
}
var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document);
_scrapDone = true;
AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
/*finally
* {
* if (_scrapBrowser != null)
* {
* _scrapBrowser.Dispose();
* _scrapBrowser = null;
* }
* }*/
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete)
{
return;
}
IEnumerable <HtmlElement> ads = null;
var results = new List <Advertisement>();
try
{
ads = _scrapBrowser.Document.ElementsByClass("div", "lot__wrapper");
if (!ads.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
return;
}
try
{
foreach (var add in ads)
{
var divDes = add.ElementsByClass("div", "lot__footer")?.FirstOrDefault();
if (divDes.InnerText.ToLower().Contains("lot is closed"))
{
continue;
}
var title = add.ElementsByClass("div", "lot__title")?.FirstOrDefault()?.InnerText;
var description = divDes?.InnerText;
var div = add.ElementsByClass("div", "lot__currentbid")?.FirstOrDefault();
var price = div.ElementsByClass("span", "lot__bidamount")?.FirstOrDefault()?.InnerText;
var a = add.ElementsByClass("a", "lot__image")?.FirstOrDefault();
var src = SafeExtractSrc(a);
if (src == null || src == " " || src == "")
{
src = "Images/noimage.png";
}
var href = SafeExtractHref(add);
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = src,
PageUrl = href,
});
}
_scrapDone = true;
AsyncScrapCompleted?.Invoke(this, results, false, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
finally
{
if (_scrapBrowser != null)
{
_scrapBrowser.Dispose();
_scrapBrowser = null;
}
}
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete)
{
return;
}
IEnumerable <HtmlElement> ads = null;
string[] filters = null;
string[] searchTextWords = _searchText.ToLower().Split(' ');
if (!string.IsNullOrEmpty(_synonyms))
{
filters = _synonyms.ToLower().Split(';');
}
var results = new List <Advertisement>();
try
{
ads = _scrapBrowser.Document.ElementsByClass("div", "row-listing");
if (!ads.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
return;
}
try
{
foreach (var add in ads)
{
var divTit = add.ElementsByClass("div", "title")?.FirstOrDefault();
if (divTit == null)
{
continue;
}
var title = divTit?.InnerText;
bool continueFlag = false;
bool breakFlag = false;
//Checking if title contains each request text word
if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower()))
{
string testTitle = title.ToLower().Replace(" ", "");
foreach (var word in searchTextWords)
{
if (!testTitle.Contains(word))
{
// Checking if title contains filters
if (filters != null && filters.Length != 0)
{
foreach (var filter in filters)
{
if (testTitle.Contains(filter) && !filter.Equals(""))
{
breakFlag = true;
}
}
}
if (breakFlag)
{
break;
}
continueFlag = true;
break;
}
}
}
if (continueFlag)
{
continue;
}
var description = add?.ElementsByClass("div", "listing-infos")?.FirstOrDefault()?.InnerText;
var price = add?.ElementsByClass("div", "listing-price")?.FirstOrDefault()?.InnerText;
if (price == null)
{
price = "POA";
}
else
{
price = price.Replace(",", "").Replace("€", "");
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
price = decimalPrice.ToString("### ###") + " €";
if (decimalPrice < _lastCostMin)
{
continue;
}
if (decimalPrice > _lastCostMax)
{
continue;
}
}
}
var divImg = add.ElementsByClass("div", "picture")?.FirstOrDefault();
var src = SafeExtratImgSrc(divImg);
if (src == null || src == " " || src == "")
{
src = "Images/noimage.png";
}
var href = SafeExtractHref(add);
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = src,
PageUrl = href,
});
}
var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document);
_scrapDone = true;
AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete)
{
return;
}
HtmlElementCollection adsli = null;
var results = new List <Advertisement>();
try
{
var ul = _scrapBrowser.Document?.ElementsByClass("ul", "lot-list").FirstOrDefault();
if (ul == null)
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
adsli = ul.GetElementsByTagName("li");
if (adsli.Count == 0)
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
return;
}
try
{
foreach (HtmlElement add in adsli)
{
var title = add.ElementsByClass("a", "title")?.FirstOrDefault()?.InnerText;
if (title == null)
{
continue;
}
var description = add.ElementsByClass("div", "lot-description")?.FirstOrDefault()?.InnerText;
var price = add.ElementsByClass("span", "time-parts")?.FirstOrDefault()?.InnerText;
var divSrc = add.ElementsByClass("div", "thumbnail")?.FirstOrDefault();
var src = SafeExtractSrc(divSrc);
if (src == null || src == " " || src == "")
{
src = "Images/noimage.png";
}
var href = SafeExtractHref(divSrc);
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = src,
PageUrl = "https://www.troostwijkauctions.com" + href,
});
}
_scrapDone = true;
AsyncScrapCompleted?.Invoke(this, results, false, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
finally
{
if (_scrapBrowser != null)
{
_scrapBrowser.Dispose();
_scrapBrowser = null;
}
}
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete)
{
return;
}
IEnumerable <HtmlElement> ads = null;
var results = new List <Advertisement>();
try
{
ads = _scrapBrowser.Document.ElementsByClass("div", "listingBox").ToList();
if (!ads.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
return;
}
try
{
foreach (var add in ads)
{
var title = add.ElementsByClass("span", "objectMaker")?.FirstOrDefault()?.InnerText + add.ElementsByClass("span", "objectTitle")?.FirstOrDefault()?.InnerText;
if (title == "" || title == null)
{
continue;
}
var description = add.ElementsByClass("span", "end")?.FirstOrDefault()?.InnerText;
var price = add.ElementsByClass("div", "highestBid")?.FirstOrDefault()?.InnerText;
if (price == null)
{
price = "POA";
}
else
{
price = price.Replace(" ", "").Replace(",", "").Replace("kr", "");
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
var euroPrice = (int)(Math.Round(decimalPrice * _currentRate));
price = euroPrice.ToString("### ###") + " €";
}
}
var styles = SafeExtractStyles(add);
styles = styles?.Remove(0, styles.IndexOf("(") + 1);
var src = styles?.Remove(styles.IndexOf(")"));
if (src == null || src == " " || src == "")
{
src = "Images/noimage.png";
}
var href = SafeExtractHref(add);
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = "https:" + src,
PageUrl = href,
});
}
_scrapDone = true;
AsyncScrapCompleted?.Invoke(this, results, false, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
finally
{
if (_scrapBrowser != null)
{
_scrapBrowser.Dispose();
_scrapBrowser = null;
}
}
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete)
{
return;
}
var results = new List <Advertisement>();
string[] filters = null;
string[] searchTextWords = _searchText.ToLower().Split(' ');
if (!string.IsNullOrEmpty(_synonyms))
{
filters = _synonyms.ToLower().Split(';');
}
IEnumerable <HtmlElement> cardsList = null;
try
{
cardsList = _scrapBrowser.Document.ElementsByClass("div", "advertisement-list-item");
if (cardsList == null && !cardsList.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
return;
}
try
{
foreach (var cardelement in cardsList)
{
var title = cardelement.ElementsByClass("a", "card-title").FirstOrDefault().InnerText;
bool continueFlag = false;
bool breakFlag = false;
//Checking if title contains each request text word
if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower()))
{
string testTitle = title.ToLower().Replace(" ", "");
foreach (var word in searchTextWords)
{
if (!testTitle.Contains(word))
{
// Checking if title contains filters
if (filters != null && filters.Length != 0)
{
foreach (var filter in filters)
{
if (testTitle.Contains(filter) && !filter.Equals(""))
{
breakFlag = true;
}
}
}
if (breakFlag)
{
break;
}
continueFlag = true;
break;
}
}
}
if (continueFlag)
{
continue;
}
var price = cardelement.ElementsByClass("p", "card-text")?.FirstOrDefault()?.InnerText;
int trashIndex = price.LastIndexOf('\n');
price = price.Substring(trashIndex + 1);
if (!string.IsNullOrWhiteSpace(price))
{
if (!Regex.IsMatch(price, @"^€[0-9]"))
{
price = "PAO";
}
else
{
price = price.Replace("Prijs: ", "").Replace(".", "").Replace(",", "").Replace("€", "").Replace("-", "").Replace(" ", "");
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
price = decimalPrice.ToString("### ###") + " €";
if (decimalPrice < _lastCostMin)
{
continue;
}
if (decimalPrice > _lastCostMax)
{
continue;
}
}
}
}
else
{
price = "PAO";
}
var description = cardelement.ElementsByClass("ul", "list-properties")?.FirstOrDefault()?.InnerText;
var src = SafeExtractSrc(cardelement);
if (src == null || src == " " || src == "")
{
src = "Images/noimage.png";
}
var href = SafeExtractHref(cardelement);
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = src,
PageUrl = href,
});
}
var pagination = _scrapBrowser.Document.ElementsByClass("ul", "pagination").FirstOrDefault();
var hasMorePages = ScrapHasMorePages(pagination);
_scrapDone = true;
AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone || _scrapBrowser.ReadyState == WebBrowserReadyState.Loading || _scrapBrowser.ReadyState == WebBrowserReadyState.Loaded || _scrapBrowser.ReadyState == WebBrowserReadyState.Uninitialized)
{
return;
}
IEnumerable <HtmlElement> ads = null;
string[] filters = null;
string[] searchTextWords = _searchText.ToLower().Split(' ');
if (!string.IsNullOrEmpty(_synonyms))
{
filters = _synonyms.ToLower().Split(';');
}
var results = new List <Advertisement>();
try
{
ads = _scrapBrowser.Document.ElementsByClass("li", "mb-20");
if (!ads.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
return;
}
try
{
foreach (HtmlElement add in ads)
{
var title = add.ElementsByClass("h3", "text-18px")?.FirstOrDefault()?.InnerText;
bool continueFlag = false;
bool breakFlag = false;
if (title == null)
{
continue;
}
//Checking if title contains each request text word
if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower()))
{
string testTitle = title.ToLower().Replace(" ", "");
foreach (var word in searchTextWords)
{
if (!testTitle.Contains(word))
{
// Checking if title contains filters
if (filters != null && filters.Length != 0)
{
foreach (var filter in filters)
{
if (testTitle.Contains(filter) && !filter.Equals(""))
{
breakFlag = true;
}
}
}
if (breakFlag)
{
break;
}
continueFlag = true;
break;
}
}
}
if (continueFlag)
{
continue;
}
var price = add.ElementsByClass("p", "price")?.FirstOrDefault()?.InnerText;
if (!string.IsNullOrWhiteSpace(price))
{
if (price.ToLower().Contains("ring for pris"))
{
price = "POA";
}
else
{
price = price.Replace("Pris:", "").Replace("DKK", "").Replace("-", "").Replace(".", "").Replace(",", "").Replace("€", "");
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
var euroPrice = (int)(Math.Round(decimalPrice * _currentRate));
price = euroPrice.ToString("### ###") + " €";
if (euroPrice < _lastCostMin)
{
continue;
}
if (euroPrice > _lastCostMax)
{
continue;
}
}
}
}
var href = SafeExtractHref(add);
var description = add.ElementsByClass("p", "description")?.FirstOrDefault()?.InnerText + Environment.NewLine + add.ElementsByClass("p", "location-info")?.FirstOrDefault()?.InnerText;
var moreDescription = add.ElementsByClass("li", "year")?.FirstOrDefault()?.InnerText;
var moremoreDescription = add.ElementsByClass("li", "hours")?.FirstOrDefault()?.InnerText;
var moremoremoreDescription = add.ElementsByClass("li", "hp")?.FirstOrDefault()?.InnerText;
description = "Year: " + moreDescription + " Hours: " + moremoreDescription + " HP: " + moremoremoreDescription + Environment.NewLine + description;
var srcDiv = add.ElementsByClass("figure", "image")?.FirstOrDefault();
var imgSrc = SafeExtractSrc(srcDiv);
if (imgSrc == null || imgSrc == " " || imgSrc == "")
{
imgSrc = "Images/noimage.png";
}
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = imgSrc,
PageUrl = href,
});
}
var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document);
_scrapDone = true;
_scrapBrowser.Stop();
AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
/*finally
* {
* if (_scrapBrowser != null)
* {
* _scrapBrowser.Dispose();
* _scrapBrowser = null;
* }
* }*/
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete)
{
return;
}
IEnumerable <HtmlElement> ads = null;
string[] filters = null;
string[] searchTextWords = _searchText.ToLower().Split(' ');
if (!string.IsNullOrEmpty(_synonyms))
{
filters = _synonyms.ToLower().Split(';');
}
var results = new List <Advertisement>();
try
{
ads = _scrapBrowser.Document.ElementsByClass("div", "mp-Listing-card-flex");
if (!ads.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
return;
}
try
{
foreach (var add in ads)
{
var descriptionDiv = add.ElementsByClass("div", "mp-Listing-card-flex-content--align-bottom")?.FirstOrDefault();
var price = descriptionDiv?.ElementsByClass("div", "mp-Listing-card-flex-price")?.FirstOrDefault()?.InnerText;
if (price == null)
{
continue;
}
var title = add.ElementsByClass("div", "mp-Listing-card-flex-title")?.FirstOrDefault()?.InnerText;
bool continueFlag = false;
bool breakFlag = false;
//Checking if title contains each request text word
if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower()))
{
string testTitle = title.ToLower().Replace(" ", "");
foreach (var word in searchTextWords)
{
if (!testTitle.Contains(word))
{
// Checking if title contains filters
if (filters != null && filters.Length != 0)
{
foreach (var filter in filters)
{
if (testTitle.Contains(filter) && !filter.Equals(""))
{
breakFlag = true;
}
}
}
if (breakFlag)
{
break;
}
continueFlag = true;
break;
}
}
}
if (continueFlag)
{
continue;
}
if (!string.IsNullOrWhiteSpace(price))
{
if (price.ToUpper().Contains("OP AANVRAAG"))
{
price = "On Request";
}
else if (price.ToUpper().Contains("BIEDEN"))
{
price = "Offer";
}
else if (price.ToUpper().Contains("N.O.T.K."))
{
price = "To be agreed";
}
else if (price.ToUpper().Contains("ZIE OMSCHRIJVING"))
{
price = "See Description";
}
else if (price.ToUpper().Contains("GERESERVEERD"))
{
price = "Reserved";
}
else
{
price = price.Replace(",", "").Replace(".", "").Replace("€", "");
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
decimalPrice /= 100;
price = decimalPrice.ToString("### ###") + " €";
if (decimalPrice < _lastCostMin)
{
continue;
}
if (decimalPrice > _lastCostMax)
{
continue;
}
}
}
}
var description = descriptionDiv.ElementsByClass("div", "mp-Listing-card-flex-location-block")?.FirstOrDefault()?.InnerText;
var srcDiv = add.ElementsByClass("div", "mp-Listing-card-flex-image-block")?.FirstOrDefault();
var src = SafeExtractSrc(srcDiv);
if (src == null || src == " " || src == "")
{
src = "Images/noimage.png";
}
var href = SafeExtractHref(add);
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = src,
PageUrl = href,
});
}
var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document);
_scrapDone = true;
AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
/*finally
* {
* if (_scrapBrowser != null)
* {
* _scrapBrowser.Dispose();
* _scrapBrowser = null;
* }
* }*/
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
try
{
string[] filters = null;
string[] searchTextWords = _searchText.ToLower().Split(' ');
string apiResponse = _scrapBrowser.Document.Body.InnerHtml;
if (apiResponse.IndexOf("json\">{") != -1)
{
apiResponse = apiResponse.Substring(apiResponse.IndexOf("json\">{"));
}
var start = apiResponse.IndexOf("{");
var end = apiResponse.LastIndexOf("}</script><script");
apiResponse = apiResponse.Substring(start, end - start + 1).Trim();
if (!string.IsNullOrEmpty(_synonyms))
{
filters = _synonyms.ToLower().Split(';');
}
var results = new List <Advertisement>();
var parsedResponse = JsonConvert.DeserializeObject <TradeMachinesSearchResponse>(apiResponse);
RequestCounter++;
if (RequestCounter > 1)
{
return;
}
if (parsedResponse == null || !parsedResponse.props.pageProps.resultsState.rawResults[0].hits.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
foreach (var add in parsedResponse.props.pageProps.resultsState.rawResults[0].hits)
{
string img;
string price;
string title;
bool continueFlag = false;
bool breakFlag = false;
if (add.product.isSparePart == true)
{
continue;
}
if (add.hasImg)
{
img = "https://dizv3061bgivy.cloudfront.net" + add.imgId;
}
else
{
img = "Images/noimage.png";
}
if (add.fr.title != null && add.product.name != null)
{
if (add.fr.title.Length >= add.product.name.Length)
{
title = add.fr.title;
}
else
{
title = add.product.name;
}
}
else
{
if (add.fr.title == null)
{
title = add.product.name;
}
else
{
title = add.fr.title;
}
}
if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower()))
{
string testTitle = title.ToLower().Replace(" ", "");
foreach (var word in searchTextWords)
{
if (!testTitle.Contains(word))
{
// Checking if title contains filters
if (filters != null && filters.Length != 0)
{
foreach (var filter in filters)
{
if (testTitle.Contains(filter) && !filter.Equals(""))
{
breakFlag = true;
}
}
}
if (breakFlag)
{
break;
}
continueFlag = true;
break;
}
}
}
if (continueFlag)
{
continue;
}
if (!add.hasPrice)
{
price = "POA";
}
else
{
price = Math.Round(add.price).ToString();
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
price = decimalPrice.ToString("### ###") + " €";
if (_lastCostMin.HasValue && decimalPrice < _lastCostMin)
{
continue;
}
if (_lastCostMax.HasValue && decimalPrice > _lastCostMax)
{
continue;
}
}
}
results.Add(new Advertisement
{
Name = title,
Description = "Year: " + add?.year +
Environment.NewLine + add.location?.country + " " + add.location?.state + " " + add.location?.city +
Environment.NewLine + add.seller?.name,
Price = price,
ImageUrl = img,
PageUrl = "https://trademachines.fr/lots/" + add?.objectID,
});
}
var parsedResponseRawResult = parsedResponse.props.pageProps.resultsState.rawResults[0];
bool hasNextPage;
if (parsedResponseRawResult.nbPages > parsedResponseRawResult.page + 1)
{
hasNextPage = true;
}
else
{
hasNextPage = false;
}
AsyncScrapCompleted?.Invoke(this, results, hasNextPage, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone)
{
return;
}
IEnumerable <HtmlElement> ads = null;
string[] filters = null;
string[] searchTextWords = _searchText.ToLower().Split(' ');
if (!string.IsNullOrEmpty(_synonyms))
{
filters = _synonyms.ToLower().Split(';');
}
var results = new List <Advertisement>();
try
{
ads = _scrapBrowser.Document.ElementsByClass("div", "machine_list_entry");
if (!ads.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
return;
}
try
{
foreach (var add in ads)
{
var title = add.ElementsByClass("span", "machine_entry_title")?.FirstOrDefault()?.InnerText;
bool continueFlag = false;
bool breakFlag = false;
//Checking if title contains each request text word
if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower()))
{
string testTitle = title.ToLower().Replace(" ", "");
foreach (var word in searchTextWords)
{
if (!testTitle.Contains(word))
{
// Checking if title contains filters
if (filters != null && filters.Length != 0)
{
foreach (var filter in filters)
{
if (testTitle.Contains(filter) && !filter.Equals(""))
{
breakFlag = true;
}
}
}
if (breakFlag)
{
break;
}
continueFlag = true;
break;
}
}
}
if (continueFlag)
{
continue;
}
var price = add.ElementsByClass("span", "machine_entry_price_price")?.FirstOrDefault()?.InnerText;
if (price == null)
{
price = add.ElementsByClass("span", "machine_entry_price_price specialOrange")?.FirstOrDefault()?.InnerText;
}
if (!string.IsNullOrWhiteSpace(price))
{
if (price.ToLower().Contains("preis auf anfrage"))
{
price = "POA";
}
if (price.ToLower().Contains("auktion"))
{
price = "Auction";
}
else
{
price = price.Replace(".", "").Replace(",", "").Replace("€", "");
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
price = decimalPrice.ToString("### ###") + " €";
if (decimalPrice < _lastCostMin)
{
continue;
}
if (decimalPrice > _lastCostMax)
{
continue;
}
}
}
}
var description = add.ElementsByClass("span", "machine_entry_description")?.FirstOrDefault()?.InnerText;
var div = add.ElementsByClass("div", "machine_entry_image")?.FirstOrDefault();
var src = SafeExtractSrc(div);
if (src != null && src[0] != 'h')
{
src = "https://www.traktorpool.de" + src;
}
else if (src == null || src == " " || src == "")
{
src = "Images/noimage.png";
}
var href = SafeExtractHref(add);
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = src,
PageUrl = href,
});
}
var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document);
_scrapDone = true;
AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
/*finally
* {
* if (_scrapBrowser != null)
* {
* _scrapBrowser.Dispose();
* _scrapBrowser = null;
* }
* }*/
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete)
{
return;
}
IEnumerable <HtmlElement> ads = null;
var results = new List <Advertisement>();
try
{
ads = _scrapBrowser.Document.ElementsByClass("div", "_3DFQ-");
if (!ads.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
return;
}
try
{
foreach (var add in ads)
{
var price = add.ElementsByClass("div", "_2OJ8g")?.FirstOrDefault()?.InnerText;
if (price == null)
{
price = "POA";
}
else
{
price = price.Replace(",", "").Replace("€", "").Replace(".", "");
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
price = decimalPrice.ToString("### ###") + " €";
if (_lastCostMin.HasValue && decimalPrice < _lastCostMin)
{
continue;
}
if (_lastCostMax.HasValue && decimalPrice > _lastCostMax)
{
continue;
}
}
}
var title = add.ElementsByClass("div", "_2tubl")?.FirstOrDefault()?.InnerText;
var description = add.ElementsByClass("div", "_32V5I").FirstOrDefault()?.InnerText;
var divImg = add.ElementsByClass("div", "LazyLoad is-visible")?.FirstOrDefault();
var src = SafeExtractSrc(divImg);
if (src == null || src == " " || src == "")
{
src = "Images/noimage.png";
}
var href = SafeExtractHref(add);
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = src,
PageUrl = href,
});
}
var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document);
_scrapDone = true;
AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
finally
{
if (_scrapBrowser != null)
{
_scrapBrowser.Dispose();
_scrapBrowser = null;
}
}
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete)
{
return;
}
//while (!_rateLoaded)
{
//System.Threading.Thread.Sleep(500);
}
var results = new List <Advertisement>();
try
{
var table = _scrapBrowser.Document.ElementsByClass("table", "productList").FirstOrDefault();
var ads = table.GetElementsByTagName("tr");
if (ads.Count == 0)
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
foreach (HtmlElement add in ads)
{
if (add == ads[0])
{
continue;
}
var price = add.ElementsByClass("TD", "col_Price")?.FirstOrDefault()?.InnerText;
if (price.ToUpper().Contains("CALL US"))
{
price = "POA";
}
else
{
price = price.Replace(",", "").Replace("£", "").Replace(" ", "");
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
var euroPrice = (int)(Math.Round(decimalPrice * _currentRate));
price = euroPrice.ToString("### ###") + " €";
if (euroPrice < _lastCostMin)
{
continue;
}
if (euroPrice > _lastCostMax)
{
continue;
}
}
}
var description = "Year: " + add.ElementsByClass("TD", "col_Year")?.FirstOrDefault()?.InnerText + "\n"
+ "Hours: " + add.ElementsByClass("TD", "col_Hours")?.FirstOrDefault()?.InnerText + "\n"
+ "Width: " + add.ElementsByClass("TD", "col_Width")?.FirstOrDefault()?.InnerText;
var hreftd = add.ElementsByClass("TD", "viewMoreButton")?.FirstOrDefault();
var href = SafeExtractHref(hreftd);
if (href != null && href[0] != 'h')
{
href = "https://www.farol.co.uk" + href;
}
var a = add.ElementsByClass("TD", "col_Image")?.FirstOrDefault();
var src = SafeExtractSrc(a);
if (src != null && src[0] != 'h')
{
src = "https://www.farol.co.uk" + src;
}
var title = add.ElementsByClass("TD", "col_Make")?.FirstOrDefault()?.InnerText + " "
+ add.ElementsByClass("TD", "col_Model")?.FirstOrDefault()?.InnerText;
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = src,
PageUrl = href,
});
}
var hasMorePages = false;
_scrapDone = true;
AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
}
private void ScrapBrowserLoadCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (_scrapDone || _scrapBrowser.ReadyState != WebBrowserReadyState.Complete)
{
return;
}
IEnumerable <HtmlElement> ads = null;
string[] filters = null;
string[] searchTextWords = _searchText.ToLower().Split(' ');
if (!string.IsNullOrEmpty(_synonyms))
{
filters = _synonyms.ToLower().Split(';');
}
var results = new List <Advertisement>();
try
{
ads = _scrapBrowser.Document.ElementsByClass("div", "item-list");
if (!ads.Any())
{
AsyncScrapCompleted?.Invoke(this, results, false, null);
return;
}
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
return;
}
try
{
foreach (var add in ads)
{
var info = add.ElementsByClass("div", "add-details")?.FirstOrDefault();
if (info == null)
{
continue;
}
var title = SafeExtrctTitle(info);
bool continueFlag = false;
bool breakFlag = false;
//Checking if title contains each request text word
if (_isFilteringActive && !title.ToLower().Contains(_searchText.ToLower()))
{
string testTitle = title.ToLower().Replace(" ", "");
foreach (var word in searchTextWords)
{
if (!testTitle.Contains(word))
{
// Checking if title contains filters
if (filters != null && filters.Length != 0)
{
foreach (var filter in filters)
{
if (testTitle.Contains(filter) && !filter.Equals(""))
{
breakFlag = true;
}
}
}
if (breakFlag)
{
break;
}
continueFlag = true;
break;
}
}
}
if (continueFlag)
{
continue;
}
var price = add.ElementsByClass("h3", "item-price-small")?.FirstOrDefault()?.InnerText;
var priceTax = add.ElementsByClass("h2", "item-price")?.FirstOrDefault()?.InnerText;
if (priceTax != null && priceTax.ToUpper() == "PRICE ON APPLICATION")
{
price = "PAO";
}
else
{
price = price.Replace(",", "").Replace("€", "").Replace(".", "").Replace("Net", "");
if (decimal.TryParse(price, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out decimal decimalPrice))
{
price = decimalPrice.ToString("### ###") + " €";
if (_lastCostMin.HasValue && decimalPrice < _lastCostMin)
{
continue;
}
if (_lastCostMax.HasValue && decimalPrice > _lastCostMax)
{
continue;
}
}
}
var description = add.ElementsByClass("span", "info-row").FirstOrDefault()?.InnerText;
var divImg = add.ElementsByClass("div", "add-image")?.FirstOrDefault();
var src = SafeExtractSrc(divImg);
if (src == "")
{
src = "http://www.pinnacleeducations.in/wp-content/uploads/2017/05/no-image.jpg";
}
var href = SafeExtractHref(add);
results.Add(new Advertisement
{
Name = title,
Description = description,
Price = price,
ImageUrl = src,
PageUrl = href,
});
}
var hasMorePages = ScrapHasMorePages(_scrapBrowser.Document);
_scrapDone = true;
AsyncScrapCompleted?.Invoke(this, results, hasMorePages, null);
}
catch (Exception ex)
{
_loggingService.LogException(ex, "Unhandled exception in ScrapBrowserLoadCompleted");
AsyncScrapCompleted?.Invoke(this, null, false, "Unhandled exception");
}
finally
{
if (_scrapBrowser != null)
{
_scrapBrowser.Dispose();
_scrapBrowser = null;
}
}
}
