public IEnumerable <Announce> Scrap(ScrapParametersDto scrapParametersDto)
{
var memory = new MemoryWriter();
var scraper = new OlxScraper
{
TypesToScrap = new [] { AnnouncementType.Sale },
Logger = new CompositeLogger
{
Loggers =
{
new ConsoleLogger()
}
},
Writer = new CompositeWriter
{
Writers =
{
new ConsoleWriter(),
memory
}
}
};
var(startPage, stopPage) = scrapParametersDto.ConvertParameters();
scraper.Start(startPage, stopPage);
scraper.ScrapeOffers();
return(memory.Announcements.Select(a => a.ToAnnounce()));
}
public static Tuple <int, int> ConvertParameters(this ScrapParametersDto parametersDto)
{
var startPage = parametersDto?.StartPage ?? 1;
var stopAfter = parametersDto?.StopAfter ?? 0;
return(new Tuple <int, int>(startPage, stopAfter));
}
public ActionResult <List <Announce> > Scrap(ScrapParametersDto scrapParametersDto)
{
var scrapedAnnounces = _scrapService.Scrap(scrapParametersDto);
var dbAnnounces = _announceService.Get(null, (PaginationFilter)null);
var newAnnounces = _streetParser.ParseStreet(_scrapService.GetOnlyNewAnnounces(dbAnnounces, scrapedAnnounces)).ToList();
foreach (var newAnnounce in newAnnounces)
{
_announceService.Create(newAnnounce);
}
return(newAnnounces);
}
