private async Task CrawlData() { try { var crawlLink = await _crawlLinkRepository.GetActive(); if (crawlLink == null) { return; } var url = crawlLink.BaseLink; if (crawlLink.FromPage.HasValue && crawlLink.ToPage.HasValue) { var page = crawlLink.FromPage > crawlLink.ToPage ? crawlLink.FromPage : crawlLink.ToPage; if (crawlLink.Finished.HasValue) { page = page - crawlLink.Finished; } url = string.Format(url, page); } var matchLinks = await GetMatchLinks(url); if (matchLinks == null) { return; } foreach (var matchLink in matchLinks) { var exsits = await _matchRepository.CheckExsits(matchLink.Slug); if (!exsits) { await SaveMatch(matchLink); } } //for test //await SaveMatch(new MatchLink //{ // Link = "http://www.fullmatchesandshows.com/2018/03/14/paris-saint-germain-vs-angers-highlights-full-match-video/", // Name = "Paris Saint Germain vs Angers – Highlights & Full Match", // ImageLink = "https://i1.wp.com/www.fullmatchesandshows.com/wp-content/uploads/2018/03/Paris-Saint-Germain-vs-Angers.jpg?w=600", // Date = "Mar 14, 2018" //}); await _crawlLinkRepository.UpdateFinished(crawlLink.Id); } catch (Exception ex) { Console.WriteLine(ex.Message); } return; }