public async Task <PaginatedListResponseData <ImageResponseData> > Handle(GetAllImagesQuery query, CancellationToken cancellationToken) { var site = await _sitesRepository.FirstOrDefault(x => x.Url.Contains(query.Request.RequestData)); if (site != null) { return(await _imagesRepository.GetFiltered <ImageResponseData>(w => w.SiteDbKey == site.DbKey, new QueryFilterRequest { SelectedPageSize = 200, CurrentPageNumber = 1 })); } return(new PaginatedListResponseData <ImageResponseData>()); }
public async Task <bool> Execute(Request <CrawlRequestData> request, CancellationToken cancellationToken) { var baseUrl = request.RequestData.Url; if (!baseUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) && !baseUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) { if (!baseUrl.StartsWith("www.")) { baseUrl = "http://www." + baseUrl; } else { baseUrl = "http://" + baseUrl; } } var client = new WebClient(); var source = client.DownloadString(new Uri(baseUrl)); var document = new HtmlDocument(); document.LoadHtml(source); var imageUrls = document.DocumentNode.Descendants("img").Select(i => i.Attributes["src"]).Select(s => s.Value).ToList(); var site = await _sitesRepository.FirstOrDefault(f => f.Url == baseUrl); if (site == null) { site = new SiteEntity { Url = baseUrl }; var dbKey = await _sitesRepository.Add(site); site = await _sitesRepository.FirstOrDefault(f => f.DbKey == dbKey); } if (site == null) { throw new Exception("Error when create entity"); } var totalImages = imageUrls.Count; var addedImages = 0; var updatedImages = 0; var omittedImages = 0; foreach (var imageUrl in imageUrls) { try { string fullImageUrl; if (imageUrl.StartsWith("//")) { fullImageUrl = "http:" + imageUrl; } else { fullImageUrl = imageUrl; } var webClient = new WebClient(); Stream stream; try { stream = await webClient.OpenReadTaskAsync(fullImageUrl); } catch (Exception e) { stream = await webClient.OpenReadTaskAsync(baseUrl + fullImageUrl); } var memoryStream = new MemoryStream(); await stream.CopyToAsync(memoryStream, cancellationToken); var imageBytes = memoryStream.ToArray(); var imageEntity = await _imagesRepository.GetFiltered <ImageEntity>(f => f.ImageUrl == fullImageUrl, new QueryFilterRequest { SelectedPageSize = 10, CurrentPageNumber = 1, OrderBy = new OrderData { FiledName = "Version", Desc = true } }); if (imageEntity.DataList.Any()) { var lastImage = imageEntity.DataList.FirstOrDefault(); if (lastImage != null) { if (lastImage.Image.ToString() != imageBytes.ToString()) { var newVersion = lastImage.Version++; long newImageVersionDbKey = await _imagesRepository.Add(new ImageEntity { ImageUrl = fullImageUrl, SiteDbKey = site.DbKey, Image = imageBytes, Version = newVersion }); updatedImages++; _logger.Debug($"Added new version ({newVersion}) of image dbKey: {newImageVersionDbKey}"); } else { omittedImages++; } } else { _logger.Debug("Error when added new image version"); } } else { long newImageDbKey = await _imagesRepository.Add(new ImageEntity { ImageUrl = fullImageUrl, SiteDbKey = site.DbKey, Image = imageBytes, Version = 0 }); addedImages++; _logger.Debug($"Added new image dbKey: {newImageDbKey}"); } } catch (Exception e) { _logger.Error(e, e.Message); } } await SendNotification(new Notification { Type = MessageTypes.Information, Message = $"Total images: <b>{totalImages}</b><br>" + $"Added new: <b>{addedImages}</b><br>" + $"Updated (create next version): <b>{updatedImages}</b><br>" + $"Omitted images: <b>{omittedImages}</b><br>" }); return(true); }