public static void Init(StringBuilder sb)
{
if (!Directory.Exists(RootFolder))
{
sb.AppendLine(string.Format("没有找到{0},创建文件夹", RootFolder));
Directory.CreateDirectory(RootFolder);
}
var lastOperationEndDate = SisDataBaseManager.GetLastOperationEndDate();
sb.AppendLine(string.Format("上次执行时间: {0}", lastOperationEndDate.ToString("yyyy-MM-dd")));
List <string> listChannel = new List <string>();
//listChannel.Add(AsiaUncensoredAuthorshipSeed);
//sb.AppendLine(string.Format("添加频道: {0}", "AsiaUncensoredAuthorshipSeed"));
//listChannel.Add(AsiaUncensoredSection);
//sb.AppendLine(string.Format("添加频道: {0}", "AsiaUncensoredSection"));
//listChannel.Add(WesternUncensoredAuthorshipSeed);
//sb.AppendLine(string.Format("添加频道: {0}", "WesternUncensoredAuthorshipSeed"));
//listChannel.Add(WesternUncensored);
//sb.AppendLine(string.Format("添加频道: {0}", "WesternUncensored"));
listChannel.Add(AsiaCensoredAuthorshipSeed);
sb.AppendLine(string.Format("添加频道: {0}", "AsiaCensoredAuthorshipSeed"));
listChannel.Add(AsiaCensoredSection);
sb.AppendLine(string.Format("添加频道: {0}", "AsiaCensoredSection"));
foreach (var channel in listChannel)
{
var page = 1;
while (page < 6)
{
var url = string.Format(channel, page);
//Console.WriteLine("Get content from " + string.Format(channel, page));
sb.AppendLine(string.Format("正在处理URL: {0}, 页码: {1}", url, page));
var res = HtmlManager.GetHtmlContentViaUrl(url, "gbk");
if (res.Success)
{
sb.AppendLine(" URL内容获取成功");
page = GetTargetThread(res.Content, ChannelMapping[channel], lastOperationEndDate, string.Format(channel, page), sb, page);
}
else
{
sb.AppendLine(" URL内容获取失败");
}
sb.AppendLine("*******************************************************************************");
page++;
}
}
}
private void SetInfo(List <FileInfo> fi)
{
URL = "";
this.ForeColor = Color.Black;
if (pictureBox1.Image != null)
{
pictureBox1.Image.Dispose();
}
if (image != null)
{
image.Dispose();
}
GC.Collect();
var fullName = fi[index].FullName;
var fileName = fi[index].Name;
textBox3.Text = fullName;
var picFolder = fullName.Replace(fi[index].Extension, "") + "/";
picsFolder = Directory.GetFiles(picFolder).Where(x => x.ToLower().Contains(".jpg")).ToList();
URL = SisDataBaseManager.GetURLFromName(fileName.Replace(fi[index].Extension, ""));
if (picFolder.Count() > 1)
{
this.ForeColor = Color.Red;
}
try
{
textBox2.Text = picsFolder[picIndex];
textBox4.Text = (picIndex + 1) + "/" + picsFolder.Count + " total " + (index + 1) + "/" + fi.Count;
image = Image.FromFile(picsFolder[picIndex]);
pictureBox1.Image = image;
}
catch (Exception e)
{
pictureBox1.Image = null;
}
}
private static int ProcessHtml(string content, string channel, DateTime lastDate, string oriUrl, StringBuilder sb, int page)
{
if (!string.IsNullOrEmpty(content))
{
List <ScanThread> temp = new List <ScanThread>();
var m = Regex.Matches(content, ListPattern, RegexOptions.Multiline | RegexOptions.IgnoreCase);
foreach (Match item in m)
{
ScanThread tempItem = new ScanThread
{
Channel = channel,
IsDownloaded = 0,
Name = FileUtility.ReplaceInvalidChar(item.Groups[4].Value),
Url = Prefix + "thread-" + item.Groups[2].Value + ".html",
ScannedDate = DateTime.Now
};
//Console.WriteLine(string.Format(" Add thread {0} url --> {1}", tempItem.Name, tempItem.Url));
sb.AppendLine(string.Format(" Add thread {0} url --> {1}", tempItem.Name, tempItem.Url));
temp.Add(tempItem);
}
foreach (var item in temp)
{
if (!SisDataBaseManager.IsExistScanThread(item))
{
SisDataBaseManager.InsertScanThread(item);
//Console.WriteLine(string.Format(" 插入帖子 {0} of channel {1} url --> {2} 日期 {3}", item.Name, item.Channel, item.Url, item.ScannedDate));
sb.AppendLine(string.Format(" 插入帖子 {0} of channel {1} url --> {2} 日期 {3}", item.Name, item.Channel, item.Url, item.ScannedDate));
}
else
{
sb.AppendLine(string.Format(" 已有此贴{0}，不再插入", item.Url));
}
}
}
return(page++);
}
public static DateTime Start(StringBuilder sb)
{
var targets = SisDataBaseManager.GetScanThread().Where(x => x.IsDownloaded != 1).OrderBy(x => x.Channel);
int current = 1;
int total = targets.Count();
sb.AppendLine(string.Format("SisDownload下载总共有{0}帖子", targets.Count()));
DateTime res = DateTime.Today;
foreach (var item in targets)
{
sb.AppendLine(string.Format("开始下载{0}", item.Url));
res = DoDownload(item, sb);
Console.WriteLine(string.Format("{0} / {1}", current, total));
current++;
}
return(res);
}
public static void Start()
{
StringBuilder sb = new StringBuilder();
try
{
sb.AppendLine(string.Format("SisDownload start at {0}", DateTime.Now.ToLongTimeString()));
ScanHelper.ScanHelper.Init(sb);
SisDataBaseManager.InsertLastOperationEndDate(DownloadHelper.DownloadHelper.Start(sb));
}
catch (Exception e)
{
sb.AppendLine(e.ToString());
}
finally
{
sb.AppendLine(string.Format("SisDownload end at {0}", DateTime.Now.ToLongTimeString()));
var root = "C:/AvLog/";
var file = "SisDownload" + DateTime.Now.ToString("yyyy-MM-dd-hh-mm-ss") + "-log.txt";
LogHelper.WriteLog(file, sb.ToString());
EmailHelper.SendEmail("SisDownloadLog", "详情见附件", new[] { "*****@*****.**" }, new[] { root + file });
}
}
public static DateTime DoParse(ScanThread st, StringBuilder sb)
{
var url = st.Url;
var res = HtmlManager.GetHtmlContentViaUrl(url, "gbk");
DateTime today = DateTime.Today;
if (res.Success)
{
sb.AppendLine(string.Format(" 获取内容成功"));
string subFolder = today.ToString("yyyy年MM月dd日") + "/" + st.Channel + "/";
if (!string.IsNullOrEmpty(res.Content))
{
var m = Regex.Matches(res.Content, DetailAttach, RegexOptions.Multiline | RegexOptions.IgnoreCase);
var attachFolder = "";
var attachName = "";
var innerSubFolder = "";
foreach (Match item in m)
{
attachFolder = FileUtility.ReplaceInvalidChar(st.Name);
attachName = FileUtility.ReplaceInvalidChar(st.Name) + ".torrent";
innerSubFolder = subFolder + attachFolder + "/";
if (!Directory.Exists(RootFolder + innerSubFolder))
{
Directory.CreateDirectory(RootFolder + innerSubFolder);
}
var attach = Prefix + DetailAttachPrefix + item.Groups[1].Value + "&clickDownload=1";
var path = RootFolder + subFolder + attachName;
//Console.WriteLine(string.Format("Download {0} to {1} and create folder {2} for picture", attach, path, innerSubFolder));
sb.AppendLine(string.Format(" Download {0} to {1} and create folder {2} for picture", attach, path, innerSubFolder));
if (!string.IsNullOrEmpty(Utils.DownloadHelper.DownloadFile(attach, path)))
{
sb.AppendLine(string.Format(" 下载附件失败"));
}
var ps = Regex.Matches(res.Content, DetailImg, RegexOptions.Multiline | RegexOptions.IgnoreCase);
int index = 1;
foreach (Match p in ps)
{
if (p.Groups[1].Value.ToLower().StartsWith("http"))
{
var pic = p.Groups[1].Value;
var picPath = RootFolder + innerSubFolder + index + ".jpg";
//Console.WriteLine(string.Format("Download Picture {0} to {1}", pic, picPath));
sb.AppendLine(string.Format(" Download Picture {0} to {1}", pic, picPath));
if (string.IsNullOrEmpty(Utils.DownloadHelper.DownloadFile(pic, picPath)))
{
sb.AppendLine(string.Format(" 下载图片失败"));
}
Console.Write(".");
index++;
}
}
sb.AppendLine(string.Format(" 更新{0}的下载状态", url));
SisDataBaseManager.UpdateDownload(url);
Console.WriteLine();
}
}
}
else
{
sb.AppendLine(string.Format(" 获取{0}内容失败", st.Url));
}
sb.AppendLine("*******************************************************************************");
return(today);
}
