public ICollection <ScrapedUser> ScrapeUsersByStrategy(int usersToScrape, IScrapingMethod scrapingStrategy) { var listOfUrls = GetFollowersURLsFromUserProfile().ToList(); var maxThreads = GetNumberOfCores() - 1; var batchSize = Convert.ToInt16(Convert.ToDouble(listOfUrls.Count) / Convert.ToDouble(maxThreads)); var listWithListsOfUrls = new List <List <string> >(); foreach (var batch in listOfUrls.Batch(batchSize)) { listWithListsOfUrls.Add(batch.ToList()); } var listOfTasks = new List <Task <ICollection <ScrapedUser> > >(); foreach (var list in listWithListsOfUrls) { var task = Task <ICollection <ScrapedUser> > .Factory.StartNew(() => GetResultsFromScraping(scrapingStrategy, list)); listOfTasks.Add(task); } Task.WaitAll(listOfTasks.ToArray()); List <ScrapedUser> results = new List <ScrapedUser>(); foreach (var task in listOfTasks) { results.AddRange(task.Result); } DriverExtensions.KillProcesses(); return(results); }
private ICollection <ScrapedUser> GetResultsFromScraping(IScrapingMethod scrapingStrategy, List <string> listOfURLs) { List <ScrapedUser> scrapeListResult = null; try { var driver = new InstaTool.MainScripts.InstagramPages.Login(); driver.PerformLogin(Login.LoggedAccount); scrapeListResult = scrapingStrategy.Scrape(listOfURLs, driver.InstaDriver).ToList(); if (scrapeListResult.Count != 0) { LogHelper.Log($"Scraped a number of {scrapeListResult.Count} users"); LogHelper.Log($"Scraped users are available to export in the dropdown below"); } else { LogHelper.Log($"{scrapeListResult.Count} users scraped"); } } catch (Exception ex) { } return(scrapeListResult); }