public ICollection <ScrapedUser> ScrapeUsersByStrategy(int usersToScrape, IScrapingMethod scrapingStrategy)
{
var listOfUrls = GetFollowersURLsFromUserProfile().ToList();
var maxThreads = GetNumberOfCores() - 1;
var batchSize = Convert.ToInt16(Convert.ToDouble(listOfUrls.Count) / Convert.ToDouble(maxThreads));
var listWithListsOfUrls = new List <List <string> >();
foreach (var batch in listOfUrls.Batch(batchSize))
{
listWithListsOfUrls.Add(batch.ToList());
}
var listOfTasks = new List <Task <ICollection <ScrapedUser> > >();
foreach (var list in listWithListsOfUrls)
{
var task = Task <ICollection <ScrapedUser> > .Factory.StartNew(() => GetResultsFromScraping(scrapingStrategy, list));
listOfTasks.Add(task);
}
Task.WaitAll(listOfTasks.ToArray());
List <ScrapedUser> results = new List <ScrapedUser>();
foreach (var task in listOfTasks)
{
results.AddRange(task.Result);
}
DriverExtensions.KillProcesses();
return(results);
}
private ICollection <ScrapedUser> GetResultsFromScraping(IScrapingMethod scrapingStrategy, List <string> listOfURLs)
{
List <ScrapedUser> scrapeListResult = null;
try
{
var driver = new InstaTool.MainScripts.InstagramPages.Login();
driver.PerformLogin(Login.LoggedAccount);
scrapeListResult = scrapingStrategy.Scrape(listOfURLs, driver.InstaDriver).ToList();
if (scrapeListResult.Count != 0)
{
LogHelper.Log($"Scraped a number of {scrapeListResult.Count} users");
LogHelper.Log($"Scraped users are available to export in the dropdown below");
}
else
{
LogHelper.Log($"{scrapeListResult.Count} users scraped");
}
}
catch (Exception ex) { }
return(scrapeListResult);
}
