public static bool HasScanUrl(ScanURL scanUrl)
{
bool res = false;
var pool = ConnectionPool.getPool();
try
{
mycon = pool.getConnection();
var sql = string.Format("Select count(url) from ScanURL where url = '{0}'", scanUrl.URL);
var reader = GetReader(sql);
while (reader.Read())
{
res = int.Parse(reader[0].ToString()) > 0 ? true : false;
}
pool.closeConnection(mycon);
}
catch (Exception e)
{
pool.closeConnection(mycon);
}
return(res);
}
public static void SaveScanUrl(ScanURL scanUrl, string page, int totalPage, int currentPage, int total, JavLibraryLog _logger)
{
var pool = ConnectionPool.getPool();
try
{
mycon = pool.getConnection();
var sql = string.Format("Insert into ScanURL values ('{0}', '{1}', '{2}', '{3}', {4}, '{5}')", scanUrl.Category.Trim(), scanUrl.URL.Trim(), scanUrl.ID.Trim(), scanUrl.Title.Trim(), scanUrl.IsDownload == true ? 1 : 0, scanUrl.CreateTime);
ExecuteQuery(sql);
pool.closeConnection(mycon);
Console.WriteLine("Processing {0} - {1} - {2}/{3} - {4}/{5} success", scanUrl.Title, scanUrl.Category, page, totalPage, currentPage, total);
_logger.WriteLog(scanUrl.URL, string.Format("Save ScanURL {0} success", scanUrl.Title));
}
catch (Exception e)
{
pool.closeConnection(mycon);
var ts = e.ToString();
if (e.ToString().Contains("Error Number:2601"))
{
Console.WriteLine("Processing {0} - {1} - {2}/{3} - {4}/{5} successfailed duplicated", scanUrl.Title, scanUrl.Category, page, totalPage, currentPage, total);
_logger.WriteExceptionLog(scanUrl.URL, string.Format("Save ScanURL {0} failed duplicated", scanUrl.Title));
}
else
{
Console.WriteLine(e.ToString());
_logger.WriteExceptionLog(scanUrl.URL, string.Format("Save ScanURL {0} falied {1}", scanUrl.Title, e.ToString()));
}
}
}
private static void ScanEachAv(ScanURL url, string status, bool force = false)
{
AV av = new AV();
if (!JavDataBaseManager.HasAv(url.URL) || force)
{
var htmlRes = JavCookieContanierHelper(url.URL);
if (htmlRes.Success)
{
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(htmlRes.Content);
av = GenerateAVModel(htmlRes.Content, url.URL);
if (!force)
{
JavDataBaseManager.InsertAV(av);
Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 插入AV => " + av.ID + " - " + av.Name + " " + status);
JavDataBaseManager.UpdateScanURL(url.URL);
Console.WriteLine("AV:" + JsonConvert.SerializeObject(new RefreshModel
{
Id = av.ID,
Name = av.Name,
Url = av.PictureURL
})
);
}
string result = "";
if (!File.Exists(ImgFolder + av.ID + av.Name + ".jpg"))
{
result = DownloadHelper.DownloadHttps(av.PictureURL, ImgFolder + av.ID + av.Name + ".jpg", "");
if (string.IsNullOrEmpty(result))
{
Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 下载AV图片成功 => " + av.ID + " - " + av.Name);
}
else
{
Console.WriteLine(result);
}
}
else
{
Console.WriteLine("已存在图片不下载");
}
}
}
else
{
Console.WriteLine("已存在 => " + url.URL + " " + status);
JavDataBaseManager.UpdateScanURL(url.URL);
}
}
private static void ScanCategoryPageUrl(string url, string cate, int current, int total, List <string> scans = null)
{
var htmlRes = JavCookieContanierHelper(url);
if (htmlRes.Success)
{
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(htmlRes.Content);
var videoPath = "//div[@class='video']";
var videoNodes = htmlDocument.DocumentNode.SelectNodes(videoPath);
if (videoNodes != null)
{
int unScanCount = 0;
foreach (var node in videoNodes)
{
var urlAndTitle = node.ChildNodes[0];
if (urlAndTitle != null && urlAndTitle.ChildNodes.Count >= 3)
{
var id = urlAndTitle.ChildNodes[0].InnerText.Trim();
var name = FileUtility.ReplaceInvalidChar(urlAndTitle.ChildNodes[2].InnerText.Trim());
var avUrl = urlAndTitle.Attributes["href"].Value.Trim().Replace("./", "http://www.javlibrary.com/cn/");
if (!string.IsNullOrEmpty(avUrl) && !string.IsNullOrEmpty(name) && !string.IsNullOrWhiteSpace(id))
{
ScanURL scan = new ScanURL
{
Category = url,
ID = id,
IsDownload = false,
Title = name,
URL = avUrl
};
if (!JavDataBaseManager.HasScan(scan))
{
unScanCount++;
JavDataBaseManager.InsertScanURL(scan);
if (scans != null)
{
scans.Add(avUrl);
}
}
}
}
}
Console.WriteLine(cate + " " + url + " 扫描了 " + unScanCount + " 未扫描, 进度" + current + " / " + total);
}
}
}
public static bool HasScan(ScanURL s)
{
var sql = @"SELECT * FROM ScanURL WHERE Url = @URL AND IsDownload = 1";
SqlParameter[] paras =
{
new SqlParameter("@URL", SqlDbType.NVarChar, 500),
};
paras[0].Value = s.URL;
return(SqlHelper.ExecuteDataTable(con, CommandType.Text, sql, paras).ToList <ScanURL>().Count > 0 ? true : false);
}
public static int InsertScanURL(ScanURL s)
{
var sql = @"INSERT INTO ScanURL (Category, URL, ID, Title, IsDownload, CreateTime) VALUES (@category, @url, @id, @title, @isDownload, @createTime)";
SqlParameter[] paras =
{
new SqlParameter("@category", SqlDbType.NVarChar, 100),
new SqlParameter("@url", SqlDbType.NVarChar, 200),
new SqlParameter("@id", SqlDbType.NVarChar, 500),
new SqlParameter("@title", SqlDbType.NVarChar, 500),
new SqlParameter("@isDownload", SqlDbType.Int),
new SqlParameter("@createTime", SqlDbType.DateTime)
};
paras[0].Value = s.Category;
paras[1].Value = s.URL;
paras[2].Value = s.ID;
paras[3].Value = s.Title;
paras[4].Value = s.IsDownload;
paras[5].Value = DateTime.Now;
return(SqlHelper.ExecuteNonQuery(con, CommandType.Text, sql, paras));
}
public static List <ScanURL> GetAllScanUrl()
{
List <ScanURL> res = new List <ScanURL>();
var pool = ConnectionPool.getPool();
try
{
mycon = pool.getConnection();
var sql = string.Format("select category, url, id, title, isdownload, createtime from scanurl");
var reader = GetReader(sql);
while (reader.Read())
{
ScanURL temp = new ScanURL();
temp.Category = reader[0].ToString();
temp.URL = reader[1].ToString();
temp.ID = reader[2].ToString();
temp.Title = reader[3].ToString();
temp.IsDownload = reader[4].ToString() == "1" ? true : false;
temp.CreateTime = DateTime.Parse(reader[5].ToString());
res.Add(temp);
}
pool.closeConnection(mycon);
}
catch (Exception e)
{
pool.closeConnection(mycon);
}
return(res);
}
private static void ScanCategoryPageUrlSingleThread(Dictionary <string, string> urls)
{
int index = 1;
foreach (var url in urls)
{
int retry = 1;
var htmlRes = new Utils.HtmlResponse();
//如果取不到cookie最多重试5次
while (retry <= 5)
{
htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", url.Key, cc);
if (htmlRes.IsExpire)
{
GetJavCookie();
retry++;
continue;
}
else
{
break;
}
}
if (htmlRes.Success)
{
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(htmlRes.Content);
var videoPath = "//div[@class='video']";
var videoNodes = htmlDocument.DocumentNode.SelectNodes(videoPath);
if (videoNodes != null)
{
int unScanCount = 0;
foreach (var node in videoNodes)
{
var urlAndTitle = node.ChildNodes[0];
if (urlAndTitle != null && urlAndTitle.ChildNodes.Count >= 3)
{
var id = urlAndTitle.ChildNodes[0].InnerText.Trim();
var name = FileUtility.ReplaceInvalidChar(urlAndTitle.ChildNodes[2].InnerText.Trim());
var avUrl = urlAndTitle.Attributes["href"].Value.Trim().Replace("./", "http://www.javlibrary.com/cn/");
if (!string.IsNullOrEmpty(avUrl) && !string.IsNullOrEmpty(name) && !string.IsNullOrWhiteSpace(id))
{
ScanURL scan = new ScanURL
{
Category = url.Value,
ID = id,
IsDownload = false,
Title = name,
URL = avUrl
};
if (!JavDataBaseManager.HasScan(scan))
{
unScanCount++;
JavDataBaseManager.InsertScanURL(scan);
}
}
}
}
Console.WriteLine(url.Value + " 第 " + index + " / " + urls.Count + " 页, 加入" + unScanCount + " 条未扫描AV");
index++;
}
}
else
{
Console.WriteLine("获取列表页 " + url.Key + " 内容失败");
}
}
}
public static RecurModel RecursiveHelper(string url, string category, int currentCategory, int totalCategories, int currentPage, CookieContainer cc, bool isUpdate = false)
{
try
{
var ret = InitHelper.InitManager.UpdateCookie(cc, url);
cc = ret.CC;
var res = ret.Content;
List <ScanURL> temp = new List <ScanURL>();
int totalPage = currentPage;
if (res.Success)
{
MatchCollection m = null;
if (isUpdate)
{
m = Regex.Matches(res.Content, updateLastPage, RegexOptions.Multiline | RegexOptions.IgnoreCase);
}
else
{
m = Regex.Matches(res.Content, listLastPage, RegexOptions.Multiline | RegexOptions.IgnoreCase);
}
if (m.Count > 0)
{
Match first = m[0];
var str = first.Groups[1].Value.Replace("\\\">", "");
totalPage = int.Parse(str.Substring(str.LastIndexOf("=") + 1));
}
Console.WriteLine(string.Format("Start process list of {0}, page {1}/{2}, categories {3}/{4}", category, currentPage, totalPage, currentCategory, totalCategories));
if (res.Success)
{
m = Regex.Matches(res.Content, listPattern, RegexOptions.Multiline | RegexOptions.IgnoreCase);
foreach (Match item in m)
{
ScanURL s = new ScanURL
{
Category = category,
CreateTime = DateTime.Now,
ID = item.Groups[3].Value,
IsDownload = false,
Title = FileUtility.ReplaceInvalidChar(item.Groups[2].Value.Replace(item.Groups[3].Value + " ", "")),
URL = item.Groups[1].Value
};
temp.Add(s);
}
foreach (var scan in temp)
{
if (!JavDataBaseManager.HasScan(scan))
{
JavDataBaseManager.InsertScanURL(scan);
}
}
if (isUpdate)
{
m = Regex.Matches(res.Content, updatePageNext, RegexOptions.Multiline | RegexOptions.IgnoreCase);
}
else
{
m = Regex.Matches(res.Content, listPageNext, RegexOptions.Multiline | RegexOptions.IgnoreCase);
}
if (m.Count > 0)
{
Match first = m[0];
if (isUpdate)
{
ForUpdate.AddRange(temp);
}
return(new RecurModel {
Url = prefix + first.Groups[1].Value,
Cc = cc
});
}
else
{
return(new RecurModel
{
Url = "",
Cc = cc
});
}
}
}
else
{
_logger.WriteExceptionLog(url, string.Format("Scan failed"));
return(new RecurModel
{
Url = "",
Cc = cc
});
}
}
catch (Exception e)
{
_logger.WriteExceptionLog(url, string.Format("Scan error {0}", e.ToString()));
}
return(new RecurModel
{
Url = url,
Cc = cc
});
}
public static bool HasScan(ScanURL entity)
{
var sql = @"SELECT * FROM ScanURL WHERE Url = @URL AND IsDownload = 1";
return(Query <ScanURL>(ConnectionStrings.Jav, sql, entity).Count > 0);
}
public static int InsertScanURL(ScanURL entity)
{
var sql = @"INSERT INTO ScanURL (Category, URL, ID, Title, IsDownload, CreateTime) VALUES (@Category, @URL, @ID, @Title, @IsDownload, GETDATE())";
return(Execute(ConnectionStrings.Jav, sql, entity));
}
