public void SaveDicMD5() { lock (this) { dic_md5 = HunterUtilities.GetMD5Hash(dictionary); SaveProject(projectPath); } }
/// <summary> /// 从UriResource获得合适的保存文件名 /// </summary> /// <param name="pInfo"></param> /// <param name="strategy"></param> /// <param name="uriRes"></param> /// <returns></returns> public static string GetFilenameFromUrl(ProjectInfo pInfo, Strategy strategy, UriResource uriRes) { /* 得到文件名的3套策略: * 1、获得超链接的标题 * 2、如果超链接的标题有乱码,则获得网络路径 * 3、如果不能正常匹配到网络路径,则以时间命名(一般不会出现这种情况) */ //删除某些标记,并Html解码,然后合法化用户名 string filepath, basicName = HunterUtilities.LegalizeFile( WebUtility.HtmlDecode(uriRes.Text) ); if (strategy.HasConfusionString(basicName)) //如果含有违禁词(如乱码) { Regex regFilename = new Regex("^http://(.*)/(?<filename>.*)\\." + strategy.Filetype); Match mFilename = regFilename.Match(uriRes.Url); //获得网络路径 if (mFilename.Success) { basicName = mFilename.Result("${filename}"); } else { basicName = DateTime.Now.ToFileTime().ToString(); } } //文件判定重复措施 filepath = Path.Combine(pInfo.filefolder, basicName + "." + strategy.Filetype); int i = 0; bool keepOriginal = true; string tempName = null; while (File.Exists(filepath)) { i++; keepOriginal = false; tempName = basicName + "_" + i.ToString(); filepath = Path.Combine(pInfo.filefolder, tempName + "." + strategy.Filetype); } ; if (!keepOriginal) { basicName = tempName; //出现重名,则在后面添加编号 } filepath = Path.Combine(pInfo.filefolder, basicName + "." + strategy.Filetype); return(filepath); }
public void CreateIPC() { if (!IPCConnected) { int state = NetworkConnection.Connect("\\\\" + HunterUtilities.GetIPFromPath(share_remote_path), "", share_usr, share_pwd); if (state != (int)NetworkConnection.ERROR_ID.ERROR_SUCCESS) { MessageBox.Show(String.Format("建立IPC连接出错,返回出错码为{0}", state), "Hunter 3", MessageBoxButtons.OK, MessageBoxIcon.Exclamation); } else { IPCConnected = true; } } }
public HunterConfig() { StrategyFolder = HunterUtilities.AbsolutePath("strategy\\"); PluginFolder = HunterUtilities.AbsolutePath("plugins\\"); CurrentStrategyFile = HunterUtilities.AbsolutePath("strategy\\bing.h3s"); UseProxy = false; PingTimeout = 3000; viewOption = new ViewOption(); notifyOption = new NotifyOption(); ProxyFilterKeywords = new List <String>(); UpdateHTTPRoot = "http://10.20.128.164:1204"; UpdateHTTPIndex = "/download/version.xml"; FailureCooldown = 60; FailureTimes = 10; HunterCore = Core.Default; }
public StrategyData LoadStrategy(String strategyPath) { StrategyData result = null; try { strategyPath = HunterUtilities.AbsolutePath(strategyPath); result = StrategyReader.LoadStrategy(strategyPath); WriteMessage(result.GetStrategyInformation()); mHunterConfig.CurrentStrategyFile = strategyPath; } catch { throw; } return(result); }
public void DisconnectIPC() { int state = NetworkConnection.Disconnect("\\\\" + HunterUtilities.GetIPFromPath(share_remote_path)); }
/// <summary> /// 下载指定uri中的所有文件。如果为null表示跳过下载,为Empty表示下载没有问题,为路径表示下载不成功。 /// </summary> /// <param name="uri">文件资源定位</param> /// <returns>下载信息</returns> public DownloadInfo DownloadFile(UriResource uriRes, Hunter h, HunterDownloadThread thisThread) { Database db = null; if (h.projectInfo.DatabaseHelper != null) { db = h.projectInfo.DatabaseHelper.GetDatabaseInstance(); } flowCalculator.Interval = 1000; flowCalculator.Elapsed += new ElapsedEventHandler(flowCalculator_Elapsed); try { HunterWebClient wc = new HunterWebClient(); #region 判断文件是否重复 bool isExist = false; //检查文件在本地是否重复 isExist = database.LinkExists(uriRes.Url); //如果文件不重复,而又为网络模式,则要检查数据库内的内容 if (!isExist && (h.projectInfo.CurrentMode == ProjectInfo.HunterMode.network)) //网络模式需要对比数据库和HunterXML { if (db == null) { hunterConsole.WriteException(new Exception("数据库连接失败,使用本地模式判重:" + uriRes.Url)); } else { bool OpenFailed = false; try { db.DbOpen(); } catch (Exception ex) { hunterConsole.WriteException(ex); } bool KRESULT = db.IsRecordExists("tb_file_infos", out isExist, new FieldValue("file_link", uriRes.Url.Replace("'", "\\'").Replace("\"", "\\\""))); if (!KRESULT || OpenFailed) { //hunterConsole.WriteException(new Exceptions("数据库连接失败,使用本地模式判重:" + uriRes.Url)); isExist = false; } try { db.DbClose(); } catch (Exception ex) { hunterConsole.WriteException(ex); } } } #endregion if (isExist) #region 链接重复对应的措施 { hunterConsole.ReportAbandonURI(uriRes, "链接重复"); return(null); } #endregion else { #region 载部分 try { hunterConsole.WriteDownload("正在下载文件:" + uriRes.Url); hunterConsole.WriteDownload( "线程ID:" + Thread.CurrentThread.ManagedThreadId + Environment.NewLine + "正在下载的文件:" + Environment.NewLine + "下载地址:" + uriRes.Url + Environment.NewLine + "下载的关键字:" + strategy.GetKeyword(uriRes.Keyword) + Environment.NewLine + "下载的页面页码:" + uriRes.index); } catch (Exception ex) { hunterConsole.WriteException(ex); } wc.DownloadFileCompleted += new System.ComponentModel.AsyncCompletedEventHandler(wc_DownloadFileCompleted); //绑定文件下载事件 wc.DownloadProgressChanged += new DownloadProgressChangedEventHandler(wc_DownloadProgressChanged); //绑定下载进度改变事件 //临时文件命名 string filepath = HunterUtilities.GetFilenameFromUrl(pInfo, strategy, uriRes); //获取合适的文件名 int timeout = 0; flowCalculator.Start(); //开始计算流量 receive1 = 0; //最开始第一次获得的数据量为0 wc.DownloadKeyword = strategy.GetKeyword(uriRes.Keyword); wc.DownloadSource = uriRes.Url; wc.DownloadDestination = filepath; if (!Directory.Exists(Path.GetDirectoryName(filepath))) { Directory.CreateDirectory(Path.GetDirectoryName(filepath)); } wc.DownloadFileAsync(new Uri(uriRes.Url), filepath); //开始下载 bool downloadProblem = false; while (wc.IsBusy) { Thread.Sleep(1000); //使用进程休眠 timeout++; if (timeout >= pInfo.timeout || h.DownloadCancelled) //如果超时或下载取消 { wc.CancelAsync(); wc.Dispose(); if (!downloadProblem) { hunterConsole.WriteDownload( "线程ID:" + Thread.CurrentThread.ManagedThreadId + Environment.NewLine + "下载超时,取消下载。"); try { hunterConsole.ReportAbandonDownloadInfo(new DownloadInfo("", uriRes.Keyword, uriRes.index, uriRes.Url, "", true, "超时", hunterProject.projectInfo.strategy.GetKeyword(uriRes.Keyword)) , "超时"); } catch { } } downloadProblem = true; } } flowCalculator.Stop(); receive1 = 0; receive2 = 0; //清空流量计 hunterConsole.outputSpeedInfo(DateTime.Now, 0); //流量计清零 wc.Dispose(); #endregion if (!downloadProblem) { //如果没有下载问题 string MD5 = string.Empty; try { MD5 = HunterUtilities.GetMD5Hash(filepath); } catch (Exception e) { hunterConsole.WriteException(new Exception("无法获取MD5。")); hunterConsole.WriteException(e); } #region 文件MD5是否重复 bool isDuplicate; //记录文件是否重复 //判断是否与本地XML重复 isDuplicate = database.isDuplicate(uriRes.Url, MD5); if (!isDuplicate && (h.projectInfo.CurrentMode == ProjectInfo.HunterMode.network)) //网络模式需要对比数据库和HunterXML { if (db == null) { hunterConsole.WriteException(new Exception("数据库连接失败,使用本地模式判重:" + MD5)); } else { bool OpenFailed = false; try { db.DbOpen(); } catch (Exception ex) { hunterConsole.WriteException(ex); } bool KRESULT = db.IsFileExists("tb_file_infos", out isDuplicate, MD5); if (!KRESULT || OpenFailed) { hunterConsole.WriteException(new Exception("数据库连接失败,使用本地模式判重:" + MD5)); isDuplicate = false; } try { db.DbClose(); } catch (Exception ex) { hunterConsole.WriteException(ex); } } } #endregion #region 文件重复、不重复对应的动作 if (!isDuplicate) { //检测是否重复。如果不重复则入库 wc.XMLFile = Path.Combine(hunterProject.projectInfo.filefolder, "$__" + Path.GetFileName(wc.DownloadDestination)) + ".xml"; try { HunterUtilities.WriteDownloadFileXML(wc.DownloadSource, wc.DownloadKeyword, Path.GetFileName(wc.DownloadDestination), (hunterProject.projectInfo.search_language == ProjectInfo.Language.none ? null : hunterProject.projectInfo.search_language.ToString()), wc.XMLFile); } catch (Exception ex) { hunterConsole.WriteException(ex); } database.addNewRecord(uriRes.Url, wc.DownloadKeyword, filepath, MD5); DownloadInfo d = new DownloadInfo(filepath, uriRes.Keyword, uriRes.index, uriRes.Url, MD5, false, "已下载", hunterProject.projectInfo.strategy.GetKeyword(uriRes.Keyword)); //Network模式:自动上传样张 if (h.projectInfo.CurrentMode == ProjectInfo.HunterMode.network) #region 网络模式上传样张 { String t_md5 = HunterUtilities.GetMD5Hash(filepath); //获得文件MD5码 String filename = t_md5 + "_" + Path.GetFileName(filepath); String combinedPath = Path.Combine(pInfo.share_remote_path, "cache", ProjectInfo.IP_ADDRESS + " (" + pInfo.name + ")"); String combinedFullPath = Path.Combine(combinedPath, filename); if (!Directory.Exists(combinedPath)) { Directory.CreateDirectory(combinedPath); } bool fileMoveSuccess = false; const int maxMoveCount = 5; int moveCount = 0; try { if (File.Exists(wc.XMLFile)) { File.Delete(Path.Combine(combinedPath, Path.GetFileName(wc.XMLFile))); File.Move(wc.XMLFile, Path.Combine(combinedPath, Path.GetFileName(wc.XMLFile))); } } catch (Exception e) { hunterConsole.WriteException(e); } while (!fileMoveSuccess) { try { if (moveCount > maxMoveCount) { break; } File.Move(filepath, combinedFullPath); fileMoveSuccess = true; } catch (Exception e) { moveCount++; hunterConsole.WriteException(e); } } } #endregion count++; hunterConsole.outputDownloadedFileNum(DateTime.Now, count); hunterConsole.ReportDownloadInfo(d); return(d); } else { DownloadInfo d = new DownloadInfo(filepath, uriRes.Keyword, uriRes.index, uriRes.Url, MD5, true, "MD5重复", hunterProject.projectInfo.strategy.GetKeyword(uriRes.Keyword)); hunterConsole.ReportAbandonDownloadInfo(d, "MD5重复"); return(d); //删除文件 } #endregion } return(new DownloadInfo(filepath, uriRes.Keyword, uriRes.index, uriRes.Url, null, true, "重复", hunterProject.projectInfo.strategy.GetKeyword(uriRes.Keyword))); //删除文件 } } catch (Exception e) { //*此处预留错误处理 hunterConsole.WriteException(e); return(null); } }
public void Start() { DownloadCancelled = false; try { if (projectInfo.mode == ProjectInfo.HunterMode.network) { projectInfo.CreateIPC(); } //记录辞典文件的MD5码 string lastMD5 = projectInfo.LoadlastDicMD5(); //读取上次保存的辞典MD5 projectInfo.SaveDicMD5(); //保存此次的辞典MD5 if (lastMD5 != HunterUtilities.GetMD5Hash(projectInfo.dictionary)) { //如果两次MD5不一致,说明辞典文件已经改变。询问是否重置辞典 if (projectInfo.strategy.CurrentSearchProgress != 0 || projectInfo.strategy.CurrentKeywordProgress != 0) { DialogResult dr = MessageBox.Show("您的辞典已经更新。要将搜索进度置零,重新开始搜索吗?", "Hunter 3", MessageBoxButtons.YesNo, MessageBoxIcon.Question); if (dr == DialogResult.Yes) { projectInfo.strategy.CurrentKeywordProgress = 0; projectInfo.strategy.CurrentSearchProgress = 0; projectInfo.strategy.RefreshProgress(0, 0); } } } } catch (Exception e) { mHunterConsole.WriteException(e); } // // //以Mode来判断是否连接服务器 try { if (projectInfo.CurrentMode == ProjectInfo.HunterMode.network) { //尝试连接一次数据库 try { projectInfo.DatabaseHelper.database.DbOpen(); projectInfo.DatabaseHelper.database.DbClose(); } catch { MessageBox.Show("连接数据库失败,本任务改为本地模式。", "Hunter 3", MessageBoxButtons.OK, MessageBoxIcon.Information); projectInfo.CurrentMode = ProjectInfo.HunterMode.local; } } isHuntingUri = true; isDownloadingUris = true; mHunterConsole.outputStartInformation(DateTime.Now, projectInfo.timeout.ToString()); for (int i = 0; i < hunterThreads.Length; i++) { if (projectInfo.CurrentMode != ProjectInfo.HunterMode.local) { hunterThreads[i].databaseHelper = new HunterDatabaseHelper(projectInfo); //hunterThreads[i].databaseHelper.connect(); } hunterThreads[i].downloadThread.Start(hunterThreads[i]); } if (mHunterConfig.UseProxy) { for (int i = 0; i < ProxyGetThreads.Length; i++) { ProxyGetThreads[i].Start(); } } thHuntUris.Start(); } catch (Exception e) { mHunterConsole.WriteException(e); } }
private void LoadStrategyList() { try { bool hasAddedSeparator = false; int addedCount = 0; DirectoryInfo d = new DirectoryInfo(HunterUtilities.AbsolutePath(mHunterConfig.StrategyFolder)); FileInfo[] files = d.GetFiles(); foreach (ToolStripMenuItem t in tsStrategies) { miStrategyList.DropDownItems.Remove(t); } foreach (FileInfo f in files) { try { StrategyData data = StrategyReader.LoadStrategy(f.FullName); //出现在列表的条件:<Title>不为空 if (data.information.Title != "") { if (!hasAddedSeparator) { ToolStripSeparator s = new ToolStripSeparator(); miStrategyList.DropDownItems.Add(s); tsStrategies.Add(s); hasAddedSeparator = true; } addedCount++; ToolStripMenuItem mi = new ToolStripMenuItem(); mi.Tag = f.FullName; mi.CheckOnClick = false; mi.Click += new EventHandler(mi_Click); mi.Text = data.information.Title; miStrategyList.DropDownItems.Add(mi); tsStrategies.Add(mi); addedCount++; if (String.Compare(f.FullName, new FileInfo(HunterUtilities.AbsolutePath(mHunterConfig.CurrentStrategyFile)).FullName, true) == 0) { mi.Checked = true; } else { mi.Checked = false; } } } catch { } } } catch (DirectoryNotFoundException ex) { mHunterConsole.WriteException(ex); mHunterConsole.WriteException(new Exception("请手动更改config.xml中的策略文件路径。")); } catch (Exception ex) { mHunterConsole.WriteException(ex); } }
private void LoadPluginList() { try { bool hasAddedSeparator = false; if (!Directory.Exists(mHunterConfig.PluginFolder)) { Directory.CreateDirectory(HunterUtilities.AbsolutePath(mHunterConfig.PluginFolder)); } DirectoryInfo d = new DirectoryInfo(mHunterConfig.PluginFolder); FileInfo[] files = d.GetFiles(); foreach (ToolStripMenuItem t in tsPlugins) { miPluginList.DropDownItems.Remove(t); } foreach (FileInfo f in files) { try { if (f.Extension != ".dll") { continue; } HunterPlugin hp = new HunterPlugin(); hp.LoadDll(f.FullName); String title = hp.GetTitle(); if (!hasAddedSeparator) { ToolStripSeparator s = new ToolStripSeparator(); miPluginList.DropDownItems.Add(s); tsPlugins.Add(s); hasAddedSeparator = true; } ToolStripMenuItem mi = new ToolStripMenuItem(); mi.Tag = hp; mi.CheckOnClick = false; mi.Click += new EventHandler((object sender, EventArgs e) => { try { new Thread(new ThreadStart(() => { HunterPlugin _hp = mi.Tag as HunterPlugin; object[] parameters = HunterPlugin.CreateArguments(_hp.GetParametersString()); if (_hp != null) { _hp.Invoke(parameters); } })).Start(); } catch { } }); mi.Text = title; miPluginList.DropDownItems.Add(mi); tsPlugins.Add(mi); } catch (Exception ex) { WriteException(ex); } } } catch (DirectoryNotFoundException ex) { mHunterConsole.WriteException(ex); mHunterConsole.WriteException(new Exception("请手动更改config.xml中的策略文件路径。")); } catch (Exception ex) { mHunterConsole.WriteException(ex); } }