private async Task <bool> ProcessTotalPageCountAsync() { if (_frontCount != null) { _totalPage = _frontCount.Value; LogHelp.Log("总分页数为:" + _totalPage); GeneratePageList(); return(true); } _firstPageStr = await HttpHelp.GetPageStringAsync(_url); if (_firstPageStr == null || _firstPageStr.Length == 0) { return(false); } var m = _regTotalPage.Match(_firstPageStr); if (m.Groups.Count != 2) { return(false); } _totalPage = int.Parse(m.Groups[1].Value); LogHelp.Log("总分页数为:" + _totalPage); GeneratePageList(); return(true); }
private async Task ProcessPageAsync(int index) { //第一页数据缓存过不需要处理 if (index == 0) { ProcessAsync(_firstPageStr ?? await HttpHelp.GetPageStringAsync(_pageList[index])); } else { ProcessAsync(await HttpHelp.GetPageStringAsync(_pageList[index])); } }
private async void ProcessDetailAsync(object stat) { try { var item = stat as Match; MoviePage pageModel = null; string movieName = ValidFileName(item.Groups[2].Value); var movieUrl = _uri.Scheme + "://" + _uri.Authority + "/bbs/" + item.Groups[1].Value; string moviedir = null; if (MyDbCOntextHelp.ExistMovie(movieUrl)) { pageModel = MyDbCOntextHelp.QueryMovie(m => m.Url == movieUrl); moviedir = Path.Combine(_baseDir, pageModel.Name); } else { pageModel = new MoviePage { CreateTime = DateTime.Now, UpdateTime = DateTime.Now, Name = movieName, Url = movieUrl, IsHandler = false, Type = _typeName, }; LogHelp.Log("影片:" + pageModel.Name); moviedir = Path.Combine(_baseDir, pageModel.Name); } if (!Directory.Exists(moviedir)) { Directory.CreateDirectory(moviedir); } if (pageModel.Id != 0 || MyDbCOntextHelp.AddPicturePage(pageModel)) { bool res = false; var detailPageString = await HttpHelp.GetPageStringAsync(pageModel.Url); if (detailPageString == null || detailPageString.Length == 0 || detailPageString.Contains("您无权进行当前操作,这可能因以下原因之一造成")) { return; } //下载次数 var m = _regDownloadCount.Match(detailPageString); if (m.Groups.Count == 2) { pageModel.DownloadCount = int.Parse(m.Groups[1].Value); } LogHelp.Log("影片下载次数:" + pageModel.DownloadCount); //图片 var imgAreaStr = _regImgArea.Match(detailPageString).Value; var mimgs = _regImg.Matches(imgAreaStr); var resList = new List <Resource>(mimgs.Count + 1); for (int j = 0; j < mimgs.Count; j++) { var imgitem = mimgs[j]; var r = new Resource { CreateTime = DateTime.Now, UpdateTime = DateTime.Now, PicturePageId = pageModel.Id, IsHandler = false, Type = 1, Url = imgitem.Groups[1].Value }; if (!r.Url.StartsWith("http")) { r.Url = _uri.Scheme + "://" + _uri.Authority + "/bbs/" + r.Url; } resList.Add(r); } LogHelp.Log("截图{0}张.....bt文件一个", mimgs.Count); //bt 检查重复 var btRes = new Resource { CreateTime = DateTime.Now, UpdateTime = DateTime.Now, PicturePageId = pageModel.Id, Type = 2, IsHandler = false, }; var p1 = detailPageString.IndexOf("检查重复</a>"); if (p1 == -1) { return; } var startbt = detailPageString.IndexOf("<a href=\"", p1); var endbt = detailPageString.IndexOf("</a>", startbt); var bta = detailPageString.Substring(startbt, endbt - startbt + 4); var mbt = _regBt.Match(bta); btRes.Name = mbt.Groups[2].Value; btRes.Url = _uri.Scheme + "://" + _uri.Authority + "/bbs/" + mbt.Groups[1].Value; resList.Add(btRes); bool isAllHandle = false; MyDbCOntextHelp.AddResourceList(resList); for (int i = 0; i < resList.Count - 1; i++) { res = await HttpHelp.DownloadImgAsync(resList[i].Url, moviedir); if (res) { } } res = await HttpHelp.DownloadFileAsync(btRes.Url, Path.Combine(moviedir, btRes.Name)); } LogHelp.Log("处理完毕:" + movieName); return; } finally { _semaphore.Release(); } }