Exemplo n.º 1
0
        public List <Ad> ParseHomePage(HtmlDocument doc, ScanPageDto scanPage)
        {
            List <Ad>          adsList = new List <Ad>();
            HtmlNodeCollection docs    = doc.DocumentNode.SelectNodes("//div[@class='container']");

            foreach (HtmlNode ad in docs)
            {
                HtmlNode nod = ad.SelectSingleNode("div[@class='title']/a");

                string title = nod.InnerText.Trim();
                string url   = scanPage.HostUrl + nod.Attributes["href"].Value;
                string idAds = url.Split('/').Last();

                HtmlNode priceTemp =
                    ad.SelectSingleNode(
                        "div[@class='info']/div[@class='price']/span[@class='value']/span[@class='amount']");
                decimal price = ScrapExtensions.ConvertStringToDecimal(priceTemp?.InnerText);

                Ad ads = Ad.Create(Guid.NewGuid(), idAds, title, url, price, scanPage.Host, scanPage.HostUrl);

                adsList.Add(ads);
            }

            return(adsList);
        }
Exemplo n.º 2
0
        public async Task delete_new_scanpage()
        {
            HttpResponseMessage response;
            var page = new ScanPageDto()
            {
                UrlAddress = "https://www.olx.pl/nieruchomosci/mieszkania/sprzedaz/warszawa/?page=2",
                Host       = "Olx",
                HostUrl    = "https://www.olx.pl/"
            };
            var payload = GetPayload(page);

            response = await Client.PostAsync(uri, payload);

            response.StatusCode.Should().BeEquivalentTo(HttpStatusCode.OK);

            var pages = await GetAllAsync <ScanPageDto>(uri);

            Assert.NotEmpty(pages);

            Guid id = pages.FirstOrDefault().Id;

            response = await Client.DeleteAsync($"{uri}/{id}");

            response.StatusCode.Should().BeEquivalentTo(HttpStatusCode.OK);
        }
Exemplo n.º 3
0
        public List <Ad> ParseHomePage(HtmlDocument doc, ScanPageDto scanPage)
        {
            List <Ad>          adsList = new List <Ad>();
            HtmlNodeCollection docs    = doc.DocumentNode.SelectNodes("//div[@class='row'] / div / article");

            foreach (HtmlNode ad in docs)
            {
                var url = ad.SelectSingleNode("div[@class='offer-item-details'] / header / h3 / a").Attributes["href"]
                          .Value;
                var title = ad.SelectSingleNode("div[@class='offer-item-details'] / header / h3 / a / span / span")
                            .InnerText.Trim();

                string idAds = ad.Attributes["data-tracking-id"].Value;

                var priceTemp =
                    ad.SelectSingleNode("div[@class='offer-item-details'] / ul / li[@class='offer-item-price']")
                    .InnerText.Trim();

                decimal price = ScrapExtensions.ConvertStringToDecimal(priceTemp);

                Ad ads = Ad.Create(Guid.NewGuid(), idAds, title, url, price, scanPage.Host, scanPage.HostUrl);

                adsList.Add(ads);
            }

            return(adsList);
        }
Exemplo n.º 4
0
        public List <Ad> ParseHomePage(HtmlDocument doc, ScanPageDto scanPage)
        {
            List <Ad>          adsList = new List <Ad>();
            HtmlNodeCollection docs    = doc.DocumentNode.SelectNodes("// tbody / tr[@class='wrap'] / td / div");

            foreach (HtmlNode ad in docs)
            {
                HtmlNode nod = ad.SelectSingleNode("table / tbody / tr[1]");

                string url = nod.SelectSingleNode("td[1] / a").Attributes["href"].Value;

                string title = nod.SelectSingleNode("td[2] / div / h3 / a / strong").InnerText.Trim();

                string idAds = ad.SelectSingleNode("table").Attributes["data-id"].Value;

                HtmlNode priceTemp = nod.SelectSingleNode("td[3] / div / p / strong");

                decimal price = ScrapExtensions.ConvertStringToDecimal(priceTemp?.InnerText);

                Ad ads = Ad.Create(Guid.NewGuid(), idAds, title, url, price, scanPage.Host, scanPage.HostUrl);

                adsList.Add(ads);
            }

            return(adsList);
        }
Exemplo n.º 5
0
        public async Task UpdateAsync(ScanPageDto page)
        {
            var scanPage = await _scanPageRepository.GetAsync(page.Id);

            if (scanPage == null)
            {
                throw new Exception($"ScanPage with id='{page.Id}' was not found.");
            }

            await _scanPageRepository.UpdateAsync(_mapper.Map <ScanPage>(page));
        }
Exemplo n.º 6
0
        public async Task add_new_scanpage()
        {
            var page = new ScanPageDto()
            {
                UrlAddress = urlAddress,
                Host       = "Gumtree",
                HostUrl    = hostUrl
            };
            var payload  = GetPayload(page);
            var response = await Client.PostAsync(uri, payload);

            response.StatusCode.Should().BeEquivalentTo(HttpStatusCode.OK);

            var pages = await GetAllAsync <ScanPageDto>(uri);

            Assert.NotEmpty(pages);

            ScanPageDto myPage = pages.FirstOrDefault(x => x.UrlAddress == urlAddress);

            myPage.Should().NotBe(null);

            //update
            myPage.Active     = false;
            myPage.UrlAddress = newUrl;

            payload  = GetPayload(myPage);
            response = await Client.PutAsync(uri, payload);

            response.StatusCode.Should().BeEquivalentTo(HttpStatusCode.OK);

            // get by id
            response = await Client.GetAsync($"{uri}/{myPage.Id}");

            var responseString = await response.Content.ReadAsStringAsync();

            var result = JsonConvert.DeserializeObject <ScanPageDto>(responseString);

            result.UrlAddress.Should().BeEquivalentTo(newUrl);

            // delete
            response = await Client.DeleteAsync($"{uri}/{result.Id}");

            response.StatusCode.Should().BeEquivalentTo(HttpStatusCode.OK);

            // get not found
            var responseNotFound = await Client.GetAsync($"{uri}/{myPage.Id}");

            var responseStringNotFound = await response.Content.ReadAsStringAsync();

            var resultNotFound = JsonConvert.DeserializeObject <ScanPageDto>(responseStringNotFound);

            resultNotFound.Should().BeNull();
        }
Exemplo n.º 7
0
        public async Task <IActionResult> Put([FromBody] ScanPageDto page)
        {
            try
            {
                await _scanPageService.UpdateAsync(page);

                return(Ok());
            }
            catch (Exception ex)
            {
                return(BadRequest(ex.Message));
            }
        }
Exemplo n.º 8
0
        public async Task add_async_should_invoke_add_async_on_repository()
        {
            // Arrange
            var         scanPageService = new ScanPageService(_scanPageRepositoryMock.Object, _mapperMock.Object);
            ScanPageDto page            = new ScanPageDto()
            {
                UrlAddress =
                    "https://www.gumtree.pl/s-mieszkania-i-domy-sprzedam-i-kupie/warszawa/page-2/v1c9073l3200008p2",
                Active  = true,
                Host    = "Gumtree",
                HostUrl = "https://www.gumtree.pl"
            };

            // Act
            await scanPageService.AddAsync(page);

            // Assert
            _scanPageRepositoryMock.Verify(x => x.AddAsync(It.IsAny <ScanPage>()), Times.Once);
        }
Exemplo n.º 9
0
 public async Task AddAsync(ScanPageDto page)
 {
     var scanPage = ScanPage.Create(Guid.NewGuid(), page.UrlAddress, page.Host, page.HostUrl, page.Active);
     await _scanPageRepository.AddAsync(scanPage);
 }
Exemplo n.º 10
0
        public async Task SeedAsync()
        {
            var users = await _userService.GetAllAsync();

            if (!users.Any())
            {
                Logger.Debug("Initializing users..");
                for (int i = 0; i <= 10; i++)
                {
                    string username = $"user{i}";
                    Logger.Debug($"Adding user: '******'.");
                    CreateUserDto newUser = new CreateUserDto
                    {
                        Email    = $"user{i}@test.com",
                        Password = "******",
                        Username = username,
                        Role     = "user"
                    };

                    await _authService.RegisterAsync(newUser);
                }
                for (int i = 0; i <= 3; i++)
                {
                    string username = $"admin{i}";
                    Logger.Debug($"Adding admin: '{username}'.");
                    CreateUserDto newUser = new CreateUserDto
                    {
                        Email    = $"admin{i}@test.com",
                        Password = "******",
                        Username = username,
                        Role     = "admin"
                    };

                    await _authService.RegisterAsync(newUser);
                }
            }
            else
            {
                Logger.Debug("Users was already initialized.");
            }

            var pages = await _scanPageService.GetAllAsync();

            if (!pages.Any())
            {
                Logger.Debug("Initializing scan pages..");
                ScanPageDto page = new ScanPageDto()
                {
                    Active     = true,
                    Host       = "Gumtree",
                    HostUrl    = "https://www.gumtree.pl",
                    UrlAddress =
                        "https://www.gumtree.pl/s-mieszkania-i-domy-sprzedam-i-kupie/warszawa/v1c9073l3200008p1"
                };
                await _scanPageService.AddAsync(page);

                ScanPageDto pageOlx = new ScanPageDto()
                {
                    Active     = true,
                    Host       = "Olx",
                    HostUrl    = "https://www.olx.pl",
                    UrlAddress = "https://www.olx.pl/nieruchomosci/mieszkania/sprzedaz/warszawa/"
                };
                await _scanPageService.AddAsync(pageOlx);

                ScanPageDto pageOtodom = new ScanPageDto()
                {
                    Active     = false,
                    Host       = "Otodom",
                    HostUrl    = "https://www.otodom.pl",
                    UrlAddress = "https://www.otodom.pl/sprzedaz/mieszkanie/warszawa/"
                };
                await _scanPageService.AddAsync(pageOtodom);
            }
            else
            {
                Logger.Debug("Scan pages was already initialized.");
            }

            var ads = await _adService.GetAllAsync();

            if (!ads.Any())
            {
                Logger.Debug($"Scraping...");
                await _scraperService.ScrapAsync();
            }
            else
            {
                Logger.Debug("Scraper was already initialized.");
            }


            Logger.Debug("Data was initialized.");
        }