private static void scrapeTask(List <Site> sites, MongoCollection <Site> mongoCollection)
{
const int sitesPerATask = 100;
const int sleepTime = 1;
int taskCount = sites.Count / sitesPerATask;
Console.WriteLine("Using {0} tasks each crawling a total of {1} sites and sleeping inbetween for {2}MS", taskCount, sitesPerATask, sleepTime);
CMCStopWatch sw = new CMCStopWatch();
sw.Start();
for (var i = 0; i < taskCount; i++)
{
var taskList = sites.Skip(i * sitesPerATask).Take(sitesPerATask);
ListProcessor(taskList.ToList(), i, taskCount, mongoCollection);
Thread.Sleep(sleepTime);
Console.WriteLine("{0} minutes remaining", sw.eta(1, taskCount).Minutes);
}
}
private static void scrapeTask(List<Site> sites, MongoCollection<Site> mongoCollection)
{
const int sitesPerATask = 100;
const int sleepTime = 1;
int taskCount = sites.Count / sitesPerATask;
Console.WriteLine("Using {0} tasks each crawling a total of {1} sites and sleeping inbetween for {2}MS", taskCount, sitesPerATask, sleepTime);
CMCStopWatch sw = new CMCStopWatch();
sw.Start();
for (var i = 0; i < taskCount; i++)
{
var taskList = sites.Skip(i * sitesPerATask).Take(sitesPerATask);
ListProcessor(taskList.ToList(), i, taskCount, mongoCollection);
Thread.Sleep(sleepTime);
Console.WriteLine("{0} minutes remaining", sw.eta(1, taskCount).Minutes);
}
}
