static void Main(string[] args)
{
Console.WriteLine("Hello World!");
#region 抓取腾讯课堂搜索架构师关键字的课程信息
//TencentCategoryEntity category = new TencentCategoryEntity()
//{
// Url = "https://ke.qq.com/course/list/%E6%9E%B6%E6%9E%84%E5%B8%88?page=1"
//};
//ISearch search = new CourseSearch(category);
//search.Crawler();
#endregion
#region 抓取腾讯课堂所有类目
//TencentCategoryEntity category = new TencentCategoryEntity()
//{
// Url = "https://ke.qq.com/course/list"
//};
//ISearch search = new CourseSearch(category);
//search.CrawlerCategory();
#endregion
#region 抓取腾讯课堂所有类目
TencentCategoryEntity category = new TencentCategoryEntity()
{
Url = "https://ke.qq.com/course/list"
};
ISearch search = new CourseSearch(category);
CrawlerAllCourse.GetAllCourse(search.CrawlerCategory());
#endregion
Console.ReadKey();
}
public static void GetAllCourse(List <TencentCategoryEntity> categorys)
{
List <TencentCategoryEntity> categoryList = categorys;
var categoryQuery = categoryList.Where(a => !a.Name.Contains("全部"));
//单线程爬虫
//foreach (var item in categoryQuery)
//{
// ISearch search = new CourseSearch(item);
// search.CrawlerAll();
//}
#region 多线程爬虫
TaskFactory taskFactory = Task.Factory;
List <Action> actions = new List <Action>();
foreach (var item in categoryQuery)
{
ISearch search = new CourseSearch(item);
actions.Add(search.CrawlerAll);
}
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 10;//最多开启10个线程
Parallel.Invoke(options, actions.ToArray());
#endregion
}
