public async Task <PaginatedListResponseData <ImageResponseData> > Handle(GetAllImagesQuery query, CancellationToken cancellationToken)
{
var site = await _sitesRepository.FirstOrDefault(x => x.Url.Contains(query.Request.RequestData));
if (site != null)
{
return(await _imagesRepository.GetFiltered <ImageResponseData>(w => w.SiteDbKey == site.DbKey, new QueryFilterRequest
{
SelectedPageSize = 200,
CurrentPageNumber = 1
}));
}
return(new PaginatedListResponseData <ImageResponseData>());
}
public async Task <bool> Execute(Request <CrawlRequestData> request, CancellationToken cancellationToken)
{
var baseUrl = request.RequestData.Url;
if (!baseUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) &&
!baseUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
{
if (!baseUrl.StartsWith("www."))
{
baseUrl = "http://www." + baseUrl;
}
else
{
baseUrl = "http://" + baseUrl;
}
}
var client = new WebClient();
var source = client.DownloadString(new Uri(baseUrl));
var document = new HtmlDocument();
document.LoadHtml(source);
var imageUrls = document.DocumentNode.Descendants("img").Select(i => i.Attributes["src"]).Select(s => s.Value).ToList();
var site = await _sitesRepository.FirstOrDefault(f => f.Url == baseUrl);
if (site == null)
{
site = new SiteEntity {
Url = baseUrl
};
var dbKey = await _sitesRepository.Add(site);
site = await _sitesRepository.FirstOrDefault(f => f.DbKey == dbKey);
}
if (site == null)
{
throw new Exception("Error when create entity");
}
var totalImages = imageUrls.Count;
var addedImages = 0;
var updatedImages = 0;
var omittedImages = 0;
foreach (var imageUrl in imageUrls)
{
try
{
string fullImageUrl;
if (imageUrl.StartsWith("//"))
{
fullImageUrl = "http:" + imageUrl;
}
else
{
fullImageUrl = imageUrl;
}
var webClient = new WebClient();
Stream stream;
try
{
stream = await webClient.OpenReadTaskAsync(fullImageUrl);
}
catch (Exception e)
{
stream = await webClient.OpenReadTaskAsync(baseUrl + fullImageUrl);
}
var memoryStream = new MemoryStream();
await stream.CopyToAsync(memoryStream, cancellationToken);
var imageBytes = memoryStream.ToArray();
var imageEntity = await _imagesRepository.GetFiltered <ImageEntity>(f => f.ImageUrl == fullImageUrl,
new QueryFilterRequest
{
SelectedPageSize = 10,
CurrentPageNumber = 1,
OrderBy = new OrderData
{
FiledName = "Version",
Desc = true
}
});
if (imageEntity.DataList.Any())
{
var lastImage = imageEntity.DataList.FirstOrDefault();
if (lastImage != null)
{
if (lastImage.Image.ToString() != imageBytes.ToString())
{
var newVersion = lastImage.Version++;
long newImageVersionDbKey = await _imagesRepository.Add(new ImageEntity
{
ImageUrl = fullImageUrl,
SiteDbKey = site.DbKey,
Image = imageBytes,
Version = newVersion
});
updatedImages++;
_logger.Debug($"Added new version ({newVersion}) of image dbKey: {newImageVersionDbKey}");
}
else
{
omittedImages++;
}
}
else
{
_logger.Debug("Error when added new image version");
}
}
else
{
long newImageDbKey = await _imagesRepository.Add(new ImageEntity
{
ImageUrl = fullImageUrl,
SiteDbKey = site.DbKey,
Image = imageBytes,
Version = 0
});
addedImages++;
_logger.Debug($"Added new image dbKey: {newImageDbKey}");
}
}
catch (Exception e)
{
_logger.Error(e, e.Message);
}
}
await SendNotification(new Notification
{
Type = MessageTypes.Information,
Message = $"Total images: <b>{totalImages}</b><br>" +
$"Added new: <b>{addedImages}</b><br>" +
$"Updated (create next version): <b>{updatedImages}</b><br>" +
$"Omitted images: <b>{omittedImages}</b><br>"
});
return(true);
}