public List <Ad> ParseHomePage(HtmlDocument doc, ScanPageDto scanPage)
{
List <Ad> adsList = new List <Ad>();
HtmlNodeCollection docs = doc.DocumentNode.SelectNodes("//div[@class='container']");
foreach (HtmlNode ad in docs)
{
HtmlNode nod = ad.SelectSingleNode("div[@class='title']/a");
string title = nod.InnerText.Trim();
string url = scanPage.HostUrl + nod.Attributes["href"].Value;
string idAds = url.Split('/').Last();
HtmlNode priceTemp =
ad.SelectSingleNode(
"div[@class='info']/div[@class='price']/span[@class='value']/span[@class='amount']");
decimal price = ScrapExtensions.ConvertStringToDecimal(priceTemp?.InnerText);
Ad ads = Ad.Create(Guid.NewGuid(), idAds, title, url, price, scanPage.Host, scanPage.HostUrl);
adsList.Add(ads);
}
return(adsList);
}
public async Task delete_new_scanpage()
{
HttpResponseMessage response;
var page = new ScanPageDto()
{
UrlAddress = "https://www.olx.pl/nieruchomosci/mieszkania/sprzedaz/warszawa/?page=2",
Host = "Olx",
HostUrl = "https://www.olx.pl/"
};
var payload = GetPayload(page);
response = await Client.PostAsync(uri, payload);
response.StatusCode.Should().BeEquivalentTo(HttpStatusCode.OK);
var pages = await GetAllAsync <ScanPageDto>(uri);
Assert.NotEmpty(pages);
Guid id = pages.FirstOrDefault().Id;
response = await Client.DeleteAsync($"{uri}/{id}");
response.StatusCode.Should().BeEquivalentTo(HttpStatusCode.OK);
}
public List <Ad> ParseHomePage(HtmlDocument doc, ScanPageDto scanPage)
{
List <Ad> adsList = new List <Ad>();
HtmlNodeCollection docs = doc.DocumentNode.SelectNodes("//div[@class='row'] / div / article");
foreach (HtmlNode ad in docs)
{
var url = ad.SelectSingleNode("div[@class='offer-item-details'] / header / h3 / a").Attributes["href"]
.Value;
var title = ad.SelectSingleNode("div[@class='offer-item-details'] / header / h3 / a / span / span")
.InnerText.Trim();
string idAds = ad.Attributes["data-tracking-id"].Value;
var priceTemp =
ad.SelectSingleNode("div[@class='offer-item-details'] / ul / li[@class='offer-item-price']")
.InnerText.Trim();
decimal price = ScrapExtensions.ConvertStringToDecimal(priceTemp);
Ad ads = Ad.Create(Guid.NewGuid(), idAds, title, url, price, scanPage.Host, scanPage.HostUrl);
adsList.Add(ads);
}
return(adsList);
}
public List <Ad> ParseHomePage(HtmlDocument doc, ScanPageDto scanPage)
{
List <Ad> adsList = new List <Ad>();
HtmlNodeCollection docs = doc.DocumentNode.SelectNodes("// tbody / tr[@class='wrap'] / td / div");
foreach (HtmlNode ad in docs)
{
HtmlNode nod = ad.SelectSingleNode("table / tbody / tr[1]");
string url = nod.SelectSingleNode("td[1] / a").Attributes["href"].Value;
string title = nod.SelectSingleNode("td[2] / div / h3 / a / strong").InnerText.Trim();
string idAds = ad.SelectSingleNode("table").Attributes["data-id"].Value;
HtmlNode priceTemp = nod.SelectSingleNode("td[3] / div / p / strong");
decimal price = ScrapExtensions.ConvertStringToDecimal(priceTemp?.InnerText);
Ad ads = Ad.Create(Guid.NewGuid(), idAds, title, url, price, scanPage.Host, scanPage.HostUrl);
adsList.Add(ads);
}
return(adsList);
}
public async Task UpdateAsync(ScanPageDto page)
{
var scanPage = await _scanPageRepository.GetAsync(page.Id);
if (scanPage == null)
{
throw new Exception($"ScanPage with id='{page.Id}' was not found.");
}
await _scanPageRepository.UpdateAsync(_mapper.Map <ScanPage>(page));
}
public async Task add_new_scanpage()
{
var page = new ScanPageDto()
{
UrlAddress = urlAddress,
Host = "Gumtree",
HostUrl = hostUrl
};
var payload = GetPayload(page);
var response = await Client.PostAsync(uri, payload);
response.StatusCode.Should().BeEquivalentTo(HttpStatusCode.OK);
var pages = await GetAllAsync <ScanPageDto>(uri);
Assert.NotEmpty(pages);
ScanPageDto myPage = pages.FirstOrDefault(x => x.UrlAddress == urlAddress);
myPage.Should().NotBe(null);
//update
myPage.Active = false;
myPage.UrlAddress = newUrl;
payload = GetPayload(myPage);
response = await Client.PutAsync(uri, payload);
response.StatusCode.Should().BeEquivalentTo(HttpStatusCode.OK);
// get by id
response = await Client.GetAsync($"{uri}/{myPage.Id}");
var responseString = await response.Content.ReadAsStringAsync();
var result = JsonConvert.DeserializeObject <ScanPageDto>(responseString);
result.UrlAddress.Should().BeEquivalentTo(newUrl);
// delete
response = await Client.DeleteAsync($"{uri}/{result.Id}");
response.StatusCode.Should().BeEquivalentTo(HttpStatusCode.OK);
// get not found
var responseNotFound = await Client.GetAsync($"{uri}/{myPage.Id}");
var responseStringNotFound = await response.Content.ReadAsStringAsync();
var resultNotFound = JsonConvert.DeserializeObject <ScanPageDto>(responseStringNotFound);
resultNotFound.Should().BeNull();
}
public async Task <IActionResult> Put([FromBody] ScanPageDto page)
{
try
{
await _scanPageService.UpdateAsync(page);
return(Ok());
}
catch (Exception ex)
{
return(BadRequest(ex.Message));
}
}
public async Task add_async_should_invoke_add_async_on_repository()
{
// Arrange
var scanPageService = new ScanPageService(_scanPageRepositoryMock.Object, _mapperMock.Object);
ScanPageDto page = new ScanPageDto()
{
UrlAddress =
"https://www.gumtree.pl/s-mieszkania-i-domy-sprzedam-i-kupie/warszawa/page-2/v1c9073l3200008p2",
Active = true,
Host = "Gumtree",
HostUrl = "https://www.gumtree.pl"
};
// Act
await scanPageService.AddAsync(page);
// Assert
_scanPageRepositoryMock.Verify(x => x.AddAsync(It.IsAny <ScanPage>()), Times.Once);
}
public async Task AddAsync(ScanPageDto page)
{
var scanPage = ScanPage.Create(Guid.NewGuid(), page.UrlAddress, page.Host, page.HostUrl, page.Active);
await _scanPageRepository.AddAsync(scanPage);
}
public async Task SeedAsync()
{
var users = await _userService.GetAllAsync();
if (!users.Any())
{
Logger.Debug("Initializing users..");
for (int i = 0; i <= 10; i++)
{
string username = $"user{i}";
Logger.Debug($"Adding user: '******'.");
CreateUserDto newUser = new CreateUserDto
{
Email = $"user{i}@test.com",
Password = "******",
Username = username,
Role = "user"
};
await _authService.RegisterAsync(newUser);
}
for (int i = 0; i <= 3; i++)
{
string username = $"admin{i}";
Logger.Debug($"Adding admin: '{username}'.");
CreateUserDto newUser = new CreateUserDto
{
Email = $"admin{i}@test.com",
Password = "******",
Username = username,
Role = "admin"
};
await _authService.RegisterAsync(newUser);
}
}
else
{
Logger.Debug("Users was already initialized.");
}
var pages = await _scanPageService.GetAllAsync();
if (!pages.Any())
{
Logger.Debug("Initializing scan pages..");
ScanPageDto page = new ScanPageDto()
{
Active = true,
Host = "Gumtree",
HostUrl = "https://www.gumtree.pl",
UrlAddress =
"https://www.gumtree.pl/s-mieszkania-i-domy-sprzedam-i-kupie/warszawa/v1c9073l3200008p1"
};
await _scanPageService.AddAsync(page);
ScanPageDto pageOlx = new ScanPageDto()
{
Active = true,
Host = "Olx",
HostUrl = "https://www.olx.pl",
UrlAddress = "https://www.olx.pl/nieruchomosci/mieszkania/sprzedaz/warszawa/"
};
await _scanPageService.AddAsync(pageOlx);
ScanPageDto pageOtodom = new ScanPageDto()
{
Active = false,
Host = "Otodom",
HostUrl = "https://www.otodom.pl",
UrlAddress = "https://www.otodom.pl/sprzedaz/mieszkanie/warszawa/"
};
await _scanPageService.AddAsync(pageOtodom);
}
else
{
Logger.Debug("Scan pages was already initialized.");
}
var ads = await _adService.GetAllAsync();
if (!ads.Any())
{
Logger.Debug($"Scraping...");
await _scraperService.ScrapAsync();
}
else
{
Logger.Debug("Scraper was already initialized.");
}
Logger.Debug("Data was initialized.");
}
