private static void Main(string[] args)
{
var conf = Configuration.ReadConfiguration("HabraMiner.cfg");
var useragent =
"";
var saver = new MongoArticleSaver<HabrArticle>(conf.DbServer, conf.DbPort, conf.DbName, conf.CollectionName, conf.BatchSave);
var tasks =
Enumerable.Range(conf.Start, conf.Count)//30466
.Select(
num =>
PageDownloadTaskFactory.CreateDownloadTask<HabrArticle>(
new Uri($"http://www.habrahabr.ru/post/{num}"), Encoding.UTF8, useragent));
var loader = new PageLoader<HabrArticle>(tasks, article => saver.Save(article));
loader.RunAllDellayedTasks(1, conf.TaskCount);
Thread.CurrentThread.Join();
}
