public async Task <IEnumerable <Media> > ProcessAsync(ScrappingSource source) { try { var downloadedMedia = new List <Media>(); var httpClient = new HttpClient(); using (var response = await httpClient.GetAsync(source.Url)) { using (var content = response.Content) { var result = await content.ReadAsStringAsync(); var htmlDocument = new HtmlDocument(); htmlDocument.LoadHtml(result); var imagesList = htmlDocument.DocumentNode.Descendants("img").Select(x => x); downloadedMedia = await this.ProccessImagesAsync(imagesList, source.Name).ConfigureAwait(false); } } var analysedMedia = await this.AnalyseDownloadedImagesAsync(downloadedMedia, source.Name).ConfigureAwait(false); await this.mediaRepository.AddManyAsync(analysedMedia.Where(m => m.Status != MediaStatus.PendingApproval)).ConfigureAwait(false); } catch (Exception ex) { throw ex; } return(Enumerable.Empty <Media>()); }
public static DomainScrapping.ScrappingSource ScrappingSourceToModel(ScrappingSource source) { return(new DomainScrapping.ScrappingSource { Name = source.Name, PaginationPattern = source.PaginationPattern, Url = source.Url.ToLowerInvariant() }); }
public async Task <ScrappingSource> InsertScrappingSourceAsync(ScrappingSource source) { if (source == null) { throw new ArgumentException(); } var sourceToInsert = ScrappingMapper.ScrappingSourceToModel(source); var insertedSource = await this.scrappingSourceService.InsertScrappingSourceAsync(sourceToInsert).ConfigureAwait(false); return(ScrappingMapper.ScrappingSourceToDto(insertedSource)); }
public async Task <ScrappingSource> InsertScrappingSourceAsync(ScrappingSource source) { if (source == null) { throw new ArgumentException(); } var exisitingSource = await this.scrappingRepository.FindAsync(s => s.Url == source.Url).ConfigureAwait(false); if (exisitingSource != null) { throw new ArgumentException("Another source with the same url already exists!"); } return(await this.scrappingRepository.AddAsync(source).ConfigureAwait(false)); }
public async Task <IActionResult> Post([FromBody] AddScrappingSourceRequest request) { try { if (!request.Url.IsValidUrl()) { return(this.BadRequest("Invalid Url format!")); } var source = new ScrappingSource() { Name = request.Name, Url = request.Url, }; return(this.Ok(await this.scrappingSourceService.InsertScrappingSourceAsync(source).ConfigureAwait(false))); } catch (Exception ex) { return(this.BadRequest(ex.Message)); } }