public static void SpiderRun() { Console.WriteLine("spider tools:youtube-dl,youget"); Console.WriteLine("please input url;"); var url = Console.ReadLine(); url = url.Trim(); var basePath = youtubeVideoPath; if (url.Contains("iqiyi.com")) { url = url + "\\"; basePath = iqiyiVideoPath; } if (url.Contains("bilibili.com")) { basePath = bilibiliVideoPath; } if (url.Contains("youku.com")) { basePath = youkuVideoPath; } if (url.Contains("qq.com")) { basePath = tencentVideoPath; } if (url.Contains("sohu.com")) { basePath = sohuVideoPath; } if (!Directory.Exists(basePath)) { Directory.CreateDirectory(basePath); } var todayDir = Path.Combine(basePath, DateTime.Now.ToString("yyyyMMdd")); if (!Directory.Exists(todayDir)) { Directory.CreateDirectory(todayDir); } var cutEnd = false; if (url.EndsWith("@")) { cutEnd = true; } url = url.Replace("@", "").Trim(); if (url.Contains("qq.com")) { VideoSpiderTools.YouGetDownLoad(url, todayDir, cutEnd); } else { VideoSpiderTools.YoutubedlDownload(url, todayDir, cutEnd); } }
private static void DownloadByUser(string url) { Console.WriteLine("请输入采集的页数:"); var numStr = Console.ReadLine(); var pageNum = Convert.ToInt32(numStr); Console.WriteLine("请输入最低播放量:"); var viewCountStr = Console.ReadLine(); var viewCount = Convert.ToInt32(viewCountStr); Console.WriteLine("开始采集"); HtmlWeb web = new HtmlWeb(); var doc = web.Load(url); var userNameNode = doc.DocumentNode.SelectSingleNode(@"//span[@id='userInfoNick']"); if (userNameNode is null) { Console.WriteLine("无法获取username,停止采集"); return; } var userName = userNameNode.InnerText.Trim(); Console.WriteLine("user name:" + userName); var userVideoPath = Path.Combine(videoDir, userName); var logPath = Path.Combine(userVideoPath, "video_list.log"); var existVideos = new List <string>(); if (!Directory.Exists(userVideoPath)) { Directory.CreateDirectory(userVideoPath); } else { if (File.Exists(logPath)) { existVideos = File.ReadLines(logPath).ToList(); } } List <string> urls = Analyse(url, pageNum, viewCount); foreach (var childUrl in urls) { Console.WriteLine("下载--" + childUrl); try { if (existVideos.Contains(childUrl.Trim())) { continue; } var task = Task.Run(() => { if (VideoSpiderTools.YouGetDownLoad(childUrl, userVideoPath, false)) { Console.WriteLine(childUrl + "--下载成功"); LogHelper.WriteLogs(childUrl.Trim(), logPath); } else { Console.WriteLine(childUrl + "--下载失败"); } }); if (!task.Wait(TimeSpan.FromMinutes(3))) { Console.WriteLine(childUrl + "--超时退出,下载失败"); } Thread.Sleep(2000); } catch (Exception ex) { Console.WriteLine(ex.Message); } } }
private static void DownloadByList(string url) { var urls = new List <string>(); Console.WriteLine("开始采集"); HtmlWeb web = new HtmlWeb(); var doc = web.Load(url); var content = NetWorkHandle.GetHtmlContent(url); var aa = content; var ulNode = doc.DocumentNode.SelectSingleNode(@"//ul[@class='figure_list']"); if (ulNode is null) { return; } var liNodes = ulNode.Descendants("li"); if (liNodes is null) { return; } foreach (var liNode in liNodes) { var aNode = liNode.Descendants("a").FirstOrDefault(); if (aNode is null) { continue; } var href = aNode.GetAttributeValue("href", ""); var childUrl = host + href; Console.WriteLine(childUrl); urls.Add(childUrl); } var todayDir = Path.Combine(basePath, DateTime.Now.ToString("yyyyMMdd")); if (!Directory.Exists(todayDir)) { Directory.CreateDirectory(todayDir); } foreach (var childUrl in urls) { Console.WriteLine("下载--" + childUrl); try { var task = Task.Run(() => { if (VideoSpiderTools.YouGetDownLoad(childUrl, todayDir, false)) { Console.WriteLine(childUrl + "--下载成功"); //LogHelper.WriteLogs(childUrl.Trim(), logPath); } else { Console.WriteLine(childUrl + "--下载失败"); } }); if (!task.Wait(TimeSpan.FromMinutes(3))) { Console.WriteLine(childUrl + "--超时退出,下载失败"); } Thread.Sleep(2000); } catch (Exception ex) { Console.WriteLine(ex.Message); } } }