public static void SpiderRun()
{
Console.WriteLine("spider tools:youtube-dl,youget");
Console.WriteLine("please input url;");
var url = Console.ReadLine();
url = url.Trim();
var basePath = youtubeVideoPath;
if (url.Contains("iqiyi.com"))
{
url = url + "\\";
basePath = iqiyiVideoPath;
}
if (url.Contains("bilibili.com"))
{
basePath = bilibiliVideoPath;
}
if (url.Contains("youku.com"))
{
basePath = youkuVideoPath;
}
if (url.Contains("qq.com"))
{
basePath = tencentVideoPath;
}
if (url.Contains("sohu.com"))
{
basePath = sohuVideoPath;
}
if (!Directory.Exists(basePath))
{
Directory.CreateDirectory(basePath);
}
var todayDir = Path.Combine(basePath, DateTime.Now.ToString("yyyyMMdd"));
if (!Directory.Exists(todayDir))
{
Directory.CreateDirectory(todayDir);
}
var cutEnd = false;
if (url.EndsWith("@"))
{
cutEnd = true;
}
url = url.Replace("@", "").Trim();
if (url.Contains("qq.com"))
{
VideoSpiderTools.YouGetDownLoad(url, todayDir, cutEnd);
}
else
{
VideoSpiderTools.YoutubedlDownload(url, todayDir, cutEnd);
}
}
private static void DownloadByUser(string url)
{
Console.WriteLine("请输入采集的页数：");
var numStr = Console.ReadLine();
var pageNum = Convert.ToInt32(numStr);
Console.WriteLine("请输入最低播放量：");
var viewCountStr = Console.ReadLine();
var viewCount = Convert.ToInt32(viewCountStr);
Console.WriteLine("开始采集");
HtmlWeb web = new HtmlWeb();
var doc = web.Load(url);
var userNameNode = doc.DocumentNode.SelectSingleNode(@"//span[@id='userInfoNick']");
if (userNameNode is null)
{
Console.WriteLine("无法获取username,停止采集");
return;
}
var userName = userNameNode.InnerText.Trim();
Console.WriteLine("user name:" + userName);
var userVideoPath = Path.Combine(videoDir, userName);
var logPath = Path.Combine(userVideoPath, "video_list.log");
var existVideos = new List <string>();
if (!Directory.Exists(userVideoPath))
{
Directory.CreateDirectory(userVideoPath);
}
else
{
if (File.Exists(logPath))
{
existVideos = File.ReadLines(logPath).ToList();
}
}
List <string> urls = Analyse(url, pageNum, viewCount);
foreach (var childUrl in urls)
{
Console.WriteLine("下载--" + childUrl);
try
{
if (existVideos.Contains(childUrl.Trim()))
{
continue;
}
var task = Task.Run(() => {
if (VideoSpiderTools.YouGetDownLoad(childUrl, userVideoPath, false))
{
Console.WriteLine(childUrl + "--下载成功");
LogHelper.WriteLogs(childUrl.Trim(), logPath);
}
else
{
Console.WriteLine(childUrl + "--下载失败");
}
});
if (!task.Wait(TimeSpan.FromMinutes(3)))
{
Console.WriteLine(childUrl + "--超时退出，下载失败");
}
Thread.Sleep(2000);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}
private static void DownloadByList(string url)
{
var urls = new List <string>();
Console.WriteLine("开始采集");
HtmlWeb web = new HtmlWeb();
var doc = web.Load(url);
var content = NetWorkHandle.GetHtmlContent(url);
var aa = content;
var ulNode = doc.DocumentNode.SelectSingleNode(@"//ul[@class='figure_list']");
if (ulNode is null)
{
return;
}
var liNodes = ulNode.Descendants("li");
if (liNodes is null)
{
return;
}
foreach (var liNode in liNodes)
{
var aNode = liNode.Descendants("a").FirstOrDefault();
if (aNode is null)
{
continue;
}
var href = aNode.GetAttributeValue("href", "");
var childUrl = host + href;
Console.WriteLine(childUrl);
urls.Add(childUrl);
}
var todayDir = Path.Combine(basePath, DateTime.Now.ToString("yyyyMMdd"));
if (!Directory.Exists(todayDir))
{
Directory.CreateDirectory(todayDir);
}
foreach (var childUrl in urls)
{
Console.WriteLine("下载--" + childUrl);
try
{
var task = Task.Run(() => {
if (VideoSpiderTools.YouGetDownLoad(childUrl, todayDir, false))
{
Console.WriteLine(childUrl + "--下载成功");
//LogHelper.WriteLogs(childUrl.Trim(), logPath);
}
else
{
Console.WriteLine(childUrl + "--下载失败");
}
});
if (!task.Wait(TimeSpan.FromMinutes(3)))
{
Console.WriteLine(childUrl + "--超时退出，下载失败");
}
Thread.Sleep(2000);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}