public async Task <IEnumerable <Media> > ProcessAsync(ScrappingSource source)
{
try
{
var downloadedMedia = new List <Media>();
var httpClient = new HttpClient();
using (var response = await httpClient.GetAsync(source.Url))
{
using (var content = response.Content)
{
var result = await content.ReadAsStringAsync();
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(result);
var imagesList = htmlDocument.DocumentNode.Descendants("img").Select(x => x);
downloadedMedia = await this.ProccessImagesAsync(imagesList, source.Name).ConfigureAwait(false);
}
}
var analysedMedia = await this.AnalyseDownloadedImagesAsync(downloadedMedia, source.Name).ConfigureAwait(false);
await this.mediaRepository.AddManyAsync(analysedMedia.Where(m => m.Status != MediaStatus.PendingApproval)).ConfigureAwait(false);
}
catch (Exception ex)
{
throw ex;
}
return(Enumerable.Empty <Media>());
}
public static DomainScrapping.ScrappingSource ScrappingSourceToModel(ScrappingSource source)
{
return(new DomainScrapping.ScrappingSource
{
Name = source.Name,
PaginationPattern = source.PaginationPattern,
Url = source.Url.ToLowerInvariant()
});
}
public async Task <ScrappingSource> InsertScrappingSourceAsync(ScrappingSource source)
{
if (source == null)
{
throw new ArgumentException();
}
var sourceToInsert = ScrappingMapper.ScrappingSourceToModel(source);
var insertedSource = await this.scrappingSourceService.InsertScrappingSourceAsync(sourceToInsert).ConfigureAwait(false);
return(ScrappingMapper.ScrappingSourceToDto(insertedSource));
}
public async Task <ScrappingSource> InsertScrappingSourceAsync(ScrappingSource source)
{
if (source == null)
{
throw new ArgumentException();
}
var exisitingSource = await this.scrappingRepository.FindAsync(s => s.Url == source.Url).ConfigureAwait(false);
if (exisitingSource != null)
{
throw new ArgumentException("Another source with the same url already exists!");
}
return(await this.scrappingRepository.AddAsync(source).ConfigureAwait(false));
}
public async Task <IActionResult> Post([FromBody] AddScrappingSourceRequest request)
{
try
{
if (!request.Url.IsValidUrl())
{
return(this.BadRequest("Invalid Url format!"));
}
var source = new ScrappingSource()
{
Name = request.Name,
Url = request.Url,
};
return(this.Ok(await this.scrappingSourceService.InsertScrappingSourceAsync(source).ConfigureAwait(false)));
}
catch (Exception ex)
{
return(this.BadRequest(ex.Message));
}
}
