private async Task CrawlData()
{
try
{
var crawlLink = await _crawlLinkRepository.GetActive();
if (crawlLink == null)
{
return;
}
var url = crawlLink.BaseLink;
if (crawlLink.FromPage.HasValue && crawlLink.ToPage.HasValue)
{
var page = crawlLink.FromPage > crawlLink.ToPage
? crawlLink.FromPage
: crawlLink.ToPage;
if (crawlLink.Finished.HasValue)
{
page = page - crawlLink.Finished;
}
url = string.Format(url, page);
}
var matchLinks = await GetMatchLinks(url);
if (matchLinks == null)
{
return;
}
foreach (var matchLink in matchLinks)
{
var exsits = await _matchRepository.CheckExsits(matchLink.Slug);
if (!exsits)
{
await SaveMatch(matchLink);
}
}
//for test
//await SaveMatch(new MatchLink
//{
// Link = "http://www.fullmatchesandshows.com/2018/03/14/paris-saint-germain-vs-angers-highlights-full-match-video/",
// Name = "Paris Saint Germain vs Angers ā Highlights & Full Match",
// ImageLink = "https://i1.wp.com/www.fullmatchesandshows.com/wp-content/uploads/2018/03/Paris-Saint-Germain-vs-Angers.jpg?w=600",
// Date = "Mar 14, 2018"
//});
await _crawlLinkRepository.UpdateFinished(crawlLink.Id);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return;
}
