private void Crawl() { List <Action> actionss = new List <Action>(); while (true) { string current = null; foreach (string url in urls.Keys) { if ((bool)urls[url]) { continue; } { current = url; } } if (current == null || count > 10) { break; } Crawlers crawlers1 = new Crawlers(current, urls, count); actionss.Add(crawlers1.crawlers); urls = crawlers1.urls; Console.WriteLine("爬行" + current + "页面!"); while (urls.Count == 1) { string html = Download(current); urls[current] = true; count++; Parse(html); } } Parallel.Invoke(actionss.ToArray()); Console.WriteLine("爬行结束"); }
private static int CrawlerCount(string x) => Crawlers.Count(y => x.ToLower().Contains(y.ToString().ToLower()));
public override bool RemoveCrawler(IProviderCrawler crawler) { return(Crawlers.Remove(crawler)); }
public void AddCrawler(IProviderCrawler crawler) { crawler.Initialize(this); Crawlers.Add(crawler); }
public JsonResult Upload(string collection = null, string ac = null) { IUpload UpOp = WebIoc.Container.Resolve <IUpload>(); var context = ControllerContext.HttpContext; if (string.IsNullOrEmpty(ac)) { ac = context.Request["action"]; } switch (ac) { case "config": return(UploadConfig()); case "uploadimage": { UploadResult result = UpOp.Upload(new UploadConfig() { AllowExtensions = CustomConfig.ImgExts.ToArray(), PathFormat = CustomConfig.EditorImgPath, SizeLimit = Convert.ToInt32(CustomConfig.CanUploadImgSize * 1024 * 1024), UploadFieldName = "upfile" }); return(Json(new { fileid = result.FileId, state = UpOp.GetStateMessage(result.State), url = result.Url, title = result.OriginFileName, original = result.OriginFileName, error = result.ErrorMessage }, "text/html")); } case "uploadscrawl": { UploadResult result = UpOp.Upload(new UploadConfig() { AllowExtensions = CustomConfig.ImgExts.ToArray(), PathFormat = CustomConfig.EditorImgPath, SizeLimit = Convert.ToInt32(CustomConfig.CanUploadImgSize * 1024 * 1024), UploadFieldName = "upfile", Base64 = true, Base64Filename = "scrawl.png" }); return(Json(new { fileid = result.FileId, state = UpOp.GetStateMessage(result.State), url = result.Url, title = result.OriginFileName, original = result.OriginFileName, error = result.ErrorMessage }, "text/html")); } case "uploadvideo": { UploadResult result = UpOp.Upload(new UploadConfig() { AllowExtensions = CustomConfig.VideoExts.ToArray(), PathFormat = CustomConfig.EditorFilePath, SizeLimit = Convert.ToInt32(CustomConfig.CanUploadFileSize * 1024 * 1024), UploadFieldName = "upfile", }); return(Json(new { fileid = result.FileId, state = UpOp.GetStateMessage(result.State), url = result.Url, title = result.OriginFileName, original = result.OriginFileName, error = result.ErrorMessage }, "text/html")); } case "uploadfile": { UploadResult result = UpOp.Upload(new UploadConfig() { AllowExtensions = CustomConfig.FileExts.ToArray(), PathFormat = CustomConfig.EditorFilePath, SizeLimit = Convert.ToInt32(CustomConfig.CanUploadFileSize * 1024 * 1024), UploadFieldName = "upfile", }); return(Json(new { fileid = result.FileId, state = UpOp.GetStateMessage(result.State), url = result.Url, title = result.OriginFileName, original = result.OriginFileName, error = result.ErrorMessage }, "text/html")); } case "listimage": { int Start = 0, Size = 20, Total = 300; ResultState State = ResultState.Success; IEnumerable <String> imgList = null; Start = String.IsNullOrEmpty(Request["start"]) ? 1 : Convert.ToInt32(Request["start"]); Size = String.IsNullOrEmpty(Request["size"]) ? 20 : Convert.ToInt32(Request["size"]); try { var path = collection; if (string.IsNullOrEmpty(collection)) { path = CustomConfig.EditorImgPath + DateTime.Now.ToString("yyyy/MM/"); } imgList = UpOp.GetFiles(path, "", Start, Size); if (string.IsNullOrEmpty(collection)) { imgList = imgList.Select(x => path + x); } } catch (UnauthorizedAccessException) { State = ResultState.AuthorizError; } catch (DirectoryNotFoundException) { State = ResultState.PathNotFound; } catch (IOException) { State = ResultState.IOError; } return(Json(new { state = GetStateString(State), list = imgList.Select(x => new { url = x }), start = Start, size = Size, total = Total }, JsonRequestBehavior.AllowGet)); } case "listfile": { int Start = 0, Size = 20, Total = 300; ResultState State = ResultState.Success; IEnumerable <String> fileList = null; Start = String.IsNullOrEmpty(Request["start"]) ? 0 : Convert.ToInt32(Request["start"]); Size = String.IsNullOrEmpty(Request["size"]) ? 20 : Convert.ToInt32(Request["size"]); try { var path = collection; if (string.IsNullOrEmpty(collection)) { path = CustomConfig.EditorImgPath + DateTime.Now.ToString("yyyy/MM/"); } fileList = UpOp.GetFiles(path, "", Start, Size); if (string.IsNullOrEmpty(collection)) { fileList = fileList.Select(x => path + x); } } catch (UnauthorizedAccessException) { State = ResultState.AuthorizError; } catch (DirectoryNotFoundException) { State = ResultState.PathNotFound; } catch (IOException) { State = ResultState.IOError; } return(Json(new { state = GetStateString(State), list = fileList.Select(x => new { url = x }), start = Start, size = Size, total = 0 }, JsonRequestBehavior.AllowGet)); } case "catchimage": { string[] Sources; CatchImg[] Crawlers; Sources = Request.Form.GetValues("source[]"); if (Sources == null || Sources.Length == 0) { return(Json(new { state = "参数错误:没有指定抓取源" })); } else { Crawlers = Sources.Select(x => new CatchImg(x, context.Server).Fetch(CustomConfig.EditorImgPath)).ToArray(); } return(Json(new { state = "SUCCESS", list = Crawlers.Select(x => new { state = x.State, source = x.SourceUrl, url = x.ServerUrl }) }, JsonRequestBehavior.AllowGet)); } default: return(Json(new { state = "action 参数为空或者 action 不被支持。" })); } }