Exemple #1
0
        /// <summary>
        ///     插入爬取的视频内容
        /// </summary>
        public static void InsertContent()
        {
            //限定20个线程运行
            ThreadPool.SetMaxThreads(20, 20);
            foreach (var type in VideoCrawer.GetVideoTypes())
            {
                var node = type;
                ThreadPool.QueueUserWorkItem(u =>
                {
                    var conn = new SqlConnection(_connectionStr);
                    //得到当前类别总分页数
                    var pageCount = VideoCrawer.GetPageCountByCode(node.Code);
                    //遍历分页得到内容后插入数据库
                    for (var i = 1; i <= pageCount; i++)
                    {
                        conn.Execute(
                            "insert VideoContent(Title,Href,ImgHref,Hits,Code,PageIndex) values(@Title,@Href,@ImgHref,@Hits,@Code,@PageIndex)",
                            VideoCrawer.GetContentsByCode(node.Code, i));

                        Console.WriteLine($"编码{node.Code}\t页码{i}\t成功");
                    }
                });
            }
        }
Exemple #2
0
        /// <summary>
        ///     插入爬取的视频类别
        /// </summary>
        public static void InsertTypes()
        {
            var conn = new SqlConnection(_connectionStr);

            conn.Execute("insert VideoType(Code,Name) values(@Code,@Name)", VideoCrawer.GetVideoTypes());
        }