public static async Task <List <CurseAddon> > Scrape(HttpClient httpClient, int tries, int from, int to) { List <CurseAddon> addons = new List <CurseAddon>(); for (int i = from; i <= to; i++) { await Task.Delay(10); string page = await DownloadPage(httpClient, tries, i); if (string.IsNullOrEmpty(page)) { logger.LogWarning("MISSED page: " + i); } else { try { List <CurseAddon> addonsFromPage = HtmlAgilityParser.FromCursePageToCurseAddons(page); addons.AddRange(addonsFromPage); logger.LogInformation("Page: " + i + ", Added: " + addonsFromPage.Count); } catch (Exception e) { logger.LogError(e, nameof(Program.Scrape)); } } } return(addons); }
public static List <Download> FromPageToDownloads(Addon addon, string page) { if (addon.ProjectUrl.Equals(Version.ELVUI)) { return(FromElvUiToDownloads(page)); } else { return(HtmlAgilityParser.FromCurseForgeToDownloads(page)); } //return new List<Download>(); }
public static IList <WebSearchCounter> ResolveWebSearchCounters() { var htmlAgilityParser = new HtmlAgilityParser(); var webClientCrawler = new WebClientCrawler(); return(new List <WebSearchCounter>() { new GoogleWebSearchCounter(htmlAgilityParser, webClientCrawler), new BingWebSearchCounter(htmlAgilityParser, webClientCrawler), new YahooWebSearchCounter(htmlAgilityParser, webClientCrawler) }); }
private static string ParsedPage(Core.Models.Addon addon, string htmlPage) { return((addon.ProjectUrl.Equals(Version.ELVUI)) ? Parse.FromElvUiPageToChanges(htmlPage) : HtmlAgilityParser.FromCursePageToChanges(htmlPage)); }