public static async Task <List <CurseAddon> > Scrape(HttpClient httpClient, int tries, int from, int to)
{
List <CurseAddon> addons = new List <CurseAddon>();
for (int i = from; i <= to; i++)
{
await Task.Delay(10);
string page = await DownloadPage(httpClient, tries, i);
if (string.IsNullOrEmpty(page))
{
logger.LogWarning("MISSED page: " + i);
}
else
{
try
{
List <CurseAddon> addonsFromPage = HtmlAgilityParser.FromCursePageToCurseAddons(page);
addons.AddRange(addonsFromPage);
logger.LogInformation("Page: " + i + ", Added: " + addonsFromPage.Count);
}
catch (Exception e)
{
logger.LogError(e, nameof(Program.Scrape));
}
}
}
return(addons);
}
public static List <Download> FromPageToDownloads(Addon addon, string page)
{
if (addon.ProjectUrl.Equals(Version.ELVUI))
{
return(FromElvUiToDownloads(page));
}
else
{
return(HtmlAgilityParser.FromCurseForgeToDownloads(page));
}
//return new List<Download>();
}
public static IList <WebSearchCounter> ResolveWebSearchCounters()
{
var htmlAgilityParser = new HtmlAgilityParser();
var webClientCrawler = new WebClientCrawler();
return(new List <WebSearchCounter>()
{
new GoogleWebSearchCounter(htmlAgilityParser, webClientCrawler),
new BingWebSearchCounter(htmlAgilityParser, webClientCrawler),
new YahooWebSearchCounter(htmlAgilityParser, webClientCrawler)
});
}
private static string ParsedPage(Core.Models.Addon addon, string htmlPage)
{
return((addon.ProjectUrl.Equals(Version.ELVUI)) ?
Parse.FromElvUiPageToChanges(htmlPage) :
HtmlAgilityParser.FromCursePageToChanges(htmlPage));
}
