/// <summary>
/// 建立数据库连接.
/// </summary>
public bool CreateConnection(string dataBasePath)
{
UnityModule.DebugPrint("正在创建数据库连接...");
try
{
//数据库连接到打开时需要断开重新连接
if (DataBaseConnection != null && DataBaseConnection.State == ConnectionState.Open)
{
CloseConnection();
UnityModule.DebugPrint("已经断开上次未关闭的数据库连接");
}
DataBaseConnection = new OleDbConnection();
DataBaseConnection.ConnectionString = string.Format("Provider=Microsoft.Jet.OLEDB.4.0;Data Source={0}", dataBasePath);
DataBaseCommand = new OleDbCommand()
{
Connection = DataBaseConnection
};
DataBaseConnection.Open();
UnityModule.DebugPrint("数据库连接创建成功！数据库状态：" + DataBaseConnection.State.ToString());
return(DataBaseConnection?.State == ConnectionState.Open);
}
catch (Exception ex)
{
UnityModule.DebugPrint("创建数据库数据库连接时失败！" + ex.Message);
return(false);
}
}
/// <summary>
/// 获取网页内容
/// </summary>
/// <param name="Pagelink">网页链接</param>
/// <returns>网页内容</returns>
private static string GetHTML(string Pagelink)
{
try
{
using (WebClient UnityWebClient = new WebClient()
{
Encoding = Encoding.UTF8
})
{
string ContentEncoding = string.Empty;
byte[] HTMLBytes;
UnityWebClient.Headers.Add("Accept-Encoding", "gzip, deflate");
UnityWebClient.BaseAddress = Pagelink;
HTMLBytes = UnityWebClient.DownloadData(Pagelink);
ContentEncoding = UnityWebClient.ResponseHeaders.Get("Content-Encoding");
if (!string.IsNullOrEmpty(ContentEncoding) && ContentEncoding.ToLower().Contains("gzip"))
{
using (GZipStream zipStream = new GZipStream(new MemoryStream(UnityWebClient.DownloadData(Pagelink)), CompressionMode.Decompress))
using (StreamReader sr = new StreamReader(zipStream, Encoding.UTF8))
return(sr.ReadToEnd());
}
else
{
return(Encoding.UTF8.GetString(HTMLBytes));
}
}
} catch (Exception ex)
{
UnityModule.DebugPrint("获取网页内容遇到异常：{0}", ex.Message);
return(string.Empty);
}
}
/// <summary>
/// 读取数据库返回值第一列第一行
/// </summary>
/// <param name="SQLCommand">数据库读取命令</param>
/// <returns>读取结果</returns>
public object ExecuteScalar(string SQLCommand)
{
if (DataBaseConnection == null)
{
UnityModule.DebugPrint("数据库连接未创建，无法读取SQL:" + SQLCommand);
return(null);
}
if (DataBaseConnection.State != ConnectionState.Open)
{
UnityModule.DebugPrint("数据库状态为：" + DataBaseConnection.State.ToString() + "；无法读取SQL:" + SQLCommand);
return(null);
}
try
{
Monitor.Enter(DataBaseCommand);
DataBaseCommand.CommandText = SQLCommand;
object DataValue = DataBaseCommand.ExecuteScalar();
Monitor.Exit(DataBaseCommand);
UnityModule.DebugPrint("命令执行成功：" + SQLCommand);
return(DataValue);
}
catch (Exception ex)
{
UnityModule.DebugPrint("读取SQL遇到错误：\n\t\t\t" + SQLCommand + "\n\t\t\t" + ex.Message);
return(null);
}
}
/// <summary>
/// 执行数据库命令
/// </summary>
/// <param name="SQLCommand">数据库命令</param>
/// <returns>执行结果</returns>
public bool ExecuteNonQuery(string SQLCommand)
{
if (DataBaseConnection == null)
{
UnityModule.DebugPrint("数据库连接未创建，无法执行SQL:" + SQLCommand);
return(false);
}
if (DataBaseConnection.State != ConnectionState.Open)
{
UnityModule.DebugPrint("数据库状态为：" + DataBaseConnection.State.ToString() + "；无法执行SQL:" + SQLCommand);
return(false);
}
try
{
Monitor.Enter(DataBaseCommand);
DataBaseCommand.CommandText = SQLCommand;
DataBaseCommand.ExecuteNonQuery();
Monitor.Exit(DataBaseCommand);
UnityModule.DebugPrint("命令执行成功：" + SQLCommand);
return(true);
}
catch (Exception ex)
{
UnityModule.DebugPrint("执行SQL遇到错误：\n\t" + SQLCommand + "\n\t" + ex.Message);
return(false);
}
}
/// <summary>
/// 扫描文章内容
/// </summary>
/// <param name="ArchiveLink">文章链接</param>
/// <returns>内容信息</returns>
private static IEnumerable <string> ScanArchive(string ArchiveLink)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("开始扫描文章：{0}", ArchiveLink);
int TempArchiveID = Convert.ToInt32(Path.GetFileName(ArchiveLink));
string ArchivePageLink = string.Empty, ArchiveString = string.Empty;
string ImagePattern = "<p><a href=\"(?<NextPageLink>.+?)\" ><img src=\"(?<ImageLink>.+?)\".*?/></a></p>";
Queue <string> ArchiveLinkQueue = new Queue <string>();
MatchCollection ArchiveMatch = null;
ArchiveLinkQueue.Enqueue(ArchiveLink);
while (ArchiveLinkQueue.Count > 0)
{
int ErrorTime = 0;
ArchivePageLink = ArchiveLinkQueue.Dequeue();
//UnityModule.DebugPrint("链接出队：{0}", ArchivePageLink);
do
{
if (ErrorTime++ > 0)
{
Thread.Sleep(1000);
}
ArchiveString = GetHTML(ArchivePageLink);
}while (string.IsNullOrEmpty(ArchiveString) && ErrorTime < 20);
if (string.IsNullOrEmpty(ArchiveString))
{
UnityModule.DebugPrint("下载页面失败多次，已跳过：{0}", ArchivePageLink);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("下载页面失败多次，已跳过：{0}", ArchivePageLink);
lock (ErrorArchiveLink)
{
ErrorArchiveLink.Add(ArchivePageLink);
}
continue;
}
ArchiveMatch = new Regex(ImagePattern, RegexOptions.IgnoreCase | RegexOptions.Singleline).Matches(ArchiveString);
foreach (Match match in ArchiveMatch)
{
ArchivePageLink = match.Groups["NextPageLink"].Value as string;
//一组照片的最后一张的链接会指向另一组照片
if (!ArchiveLinkQueue.Contains(ArchivePageLink))
{
ArchiveLinkQueue.Enqueue(ArchivePageLink);
}
yield return(match.Groups["ImageLink"].Value as string);
if (!ArchivePageLink.StartsWith(ArchiveLink))
{
yield break;
}
//UnityModule.DebugPrint("发现新链接：{0}", ArchivePageLink);
}
}
yield break;
}
/// <summary>
/// 关闭数据库连接
/// </summary>
public void CloseConnection()
{
UnityModule.DebugPrint("关闭数据库连接...");
try
{
if (DataBaseConnection == null)
{
return;
}
DataBaseConnection?.Close();
DataBaseConnection?.Dispose();
DataBaseCommand?.Dispose();
DataBaseConnection = null;
DataBaseCommand = null;
}
catch (Exception ex) {
UnityModule.DebugPrint("关闭数据库连接时遇到错误：", ex.Message);
}
}
/// <summary>
/// 读取数据库至数据适配器
/// </summary>
/// <param name="SQLCommand">数据库读取命令</param>
/// <returns>读取结果</returns>
public OleDbDataAdapter ExecuteAdapter(string SQLCommand)
{
if (DataBaseConnection == null)
{
UnityModule.DebugPrint("数据库连接未创建，无法读取SQL:" + SQLCommand);
return(null);
}
try
{
Monitor.Enter(DataBaseCommand);
OleDbDataAdapter DataAdapter = new OleDbDataAdapter(SQLCommand, DataBaseConnection);
Monitor.Exit(DataBaseCommand);
UnityModule.DebugPrint("命令执行成功：" + SQLCommand);
return(DataAdapter);
}
catch (Exception ex)
{
UnityModule.DebugPrint("读取SQL遇到错误：\n\t" + SQLCommand + "\n\t" + ex.Message);
return(null);
}
}
/// <summary>
/// 下载文章
/// </summary>
/// <param name="ArchivePacket">文章数据包</param>
private static void DownloadArchive(ArchiveModel ArchivePacket)
{
string ImagePath = string.Empty, ArchiveDirectory = string.Empty, TempTitle = ArchivePacket.Title;
TempTitle = TempTitle.Replace("?", "_w").Replace(":", "_m").Replace("\\", "_").Replace("/", "_f").Replace("|", "_s").Replace("*", "_x");
ArchiveDirectory = Path.Combine(UnityModule.ContentDirectory, TempTitle);
try
{
if (!Directory.Exists(ArchiveDirectory))
{
Console.WriteLine("创建新目录：{0}", ArchiveDirectory);
Directory.CreateDirectory(ArchiveDirectory);
}
}
catch (Exception ex)
{
System.Diagnostics.Debug.Print(ArchiveDirectory);
Console.WriteLine("创建文章目录失败：{0} / {1}", ArchiveDirectory, ex.Message);
}
UnityDBController.ExecuteNonQuery("DELETE FROM ImageBase WHERE ArchiveID = {0} ;", ArchivePacket.ArchiveID);
foreach (string ImageLink in ScanArchive(ArchivePacket.ArchiveLink))
{
ImagePath = Path.Combine(ArchiveDirectory, Path.GetFileName(ImageLink));
if (UnityDBController.ExecuteNonQuery("INSERT INTO ImageBase (ArchiveID, ImageLink, ImagePath) VALUES({0}, '{1}', '{2}') ;",
ArchivePacket.ArchiveID, ImageLink, ImagePath))
{
if (!File.Exists(ImagePath))
{
using (WebClient DownloadWebClient = new WebClient()
{
Encoding = Encoding.UTF8
})
{
try
{
//绕过防盗链（使用 Fiddler4 对比盗链和非盗链的HTTP请求头信息即可）
DownloadWebClient.Headers.Add(HttpRequestHeader.Referer, ArchivePacket.ArchiveLink);
DownloadWebClient.DownloadFile(ImageLink, ImagePath);
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("图像下载成功：{0}", ImagePath);
UnityModule.DebugPrint("图像下载成功：{0}", ImagePath);
}
catch (Exception ex)
{
UnityModule.DebugPrint("下载图像遇到错误：{0} / {1}", ImageLink, ex.Message);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("下载图像遇到错误：{0} / {1}", ImageLink, ex.Message);
lock (ErrorImageLink)
{
ErrorImageLink.Add(ImageLink);
}
}
}
}
else
{
Console.ForegroundColor = ConsoleColor.DarkGray;
Console.WriteLine("已经存在的图像：{0}", ImagePath);
}
}
else
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("图像记录插入数据仓库失败：{0}", ImageLink);
}
}
Console.ForegroundColor = ConsoleColor.Magenta;
Console.WriteLine("文章下载完成：{0}", ArchivePacket.Title);
}
public void Dispose()
{
UnityModule.DebugPrint("数据库连接对象释放...");
CloseConnection();
}
