/// <summary> /// get keyworld content from text file /// </summary> /// <param name="url"></param> /// <returns></returns> public List <KeyWorldEntity> getTextContent(string url) { List <KeyWorldEntity> listkeyWorldEntity = new List <KeyWorldEntity>(); try { if (string.IsNullOrEmpty(url)) { return(null); } IOFileHelper iOFileHelper = new IOFileHelper(); var strContext = iOFileHelper.ReadTxtFile(url); var byteData = strContext.Split('-'); foreach (var item in byteData) { KeyWorldEntity keyWorldEntity = new KeyWorldEntity(); keyWorldEntity.KeyWorld = item.Trim(); listkeyWorldEntity.Add(keyWorldEntity); } return(listkeyWorldEntity); } catch { Debug.WriteLine("KeyWorldService"); } return(listkeyWorldEntity); }
/// <summary> /// read json file to list /// </summary> /// <param name="url"></param> /// <returns></returns> public List <BarNameEntity> getJsonContent(string filepath) { List <BarNameEntity> listkeyWorldEntity = new List <BarNameEntity>(); try { if (string.IsNullOrEmpty(filepath)) { return(null); } IOFileHelper iOFileHelper = new IOFileHelper(); var strContext = iOFileHelper.ReadTxtFile(filepath); var byteData = strContext.Split('-'); UrlCodeConvert urlCodeConvert = new UrlCodeConvert(); foreach (var item in byteData) { BarNameEntity barNameEntity = new BarNameEntity(); barNameEntity.BarName = item.Trim(); barNameEntity.Url = urlCodeConvert.EncodeUrl(item.Trim()); listkeyWorldEntity.Add(barNameEntity); } return(listkeyWorldEntity); } catch { Debug.WriteLine("KeyWorldService"); } return(listkeyWorldEntity); }
/// <summary> /// 读取applocal json 文件贴吧Name /// </summary> /// <param name="JsonPath"></param> public static List <SearchlistEntity> ReadTBNameFile(Uri JsonPath = null) { try { List <SearchlistEntity> listSearchlistEntity = new List <SearchlistEntity>(); IOFileHelper iOFileHelper = new IOFileHelper(); var strJson = iOFileHelper.ReadJsonFileToString(@"/CrawlerNproData/BarName.json"); JsonHelper jsonHelper = new JsonHelper(); var listEntity = jsonHelper.DeserializeJsonTo <List <BarNameEntity> >(strJson); UrlCodeConvert ucc = new UrlCodeConvert(); foreach (var item in listEntity) { SearchlistEntity searchlistEntity = new SearchlistEntity(); searchlistEntity.Title = item.BarName; searchlistEntity.SearchContent = ucc.DecodeUrl(item.Url); listSearchlistEntity.Add(searchlistEntity); } return(listSearchlistEntity); } catch { Debug.WriteLine("-------cls-CrawlerNproView------func ReadTBNameFile----"); } return(null); }
///java sys /// <summary> /// read database to local json file /// </summary> public void SaveTBNameToJson(Uri JsonPath = null) { //读取数据库 SysTBNameDAL sysTBNAMEDAL = new SysTBNameDAL(); var listTBName = sysTBNAMEDAL.SelectTBName(null);//token //转换json JsonHelper jsonHelper = new JsonHelper(); var strJson = jsonHelper.SerializerJson(listTBName); //保存文件 IOFileHelper iOFileHelper = new IOFileHelper(); string JsonContentPath = ConfigurationManager.AppSettings["JsonContentPath"]; iOFileHelper.SaveJsonFile(JsonContentPath, "BarName.json", strJson);//文件名写死 }
public ActionResult ShowImage(int globalId, string documentName, int type = 1) { const string fileBasePath = Constants.ImagePath.ImageFolderPath; string filePath = string.Empty; if (!string.IsNullOrEmpty(documentName) && globalId > 0) { string physicalPath = System.Web.HttpContext.Current.Server.MapPath(fileBasePath) + documentName; if (System.IO.File.Exists(physicalPath)) { filePath = Url.Content(fileBasePath + documentName); } else { var service = DependencyResolver.Current.GetService(typeof(IDocumentInformationService)) as IDocumentInformationService; var documentInformation = service.Get(new DocumentInformation() { GlobalId = globalId }); if (documentInformation != null && !string.IsNullOrEmpty(documentInformation.DocumentName)) { documentName = documentInformation.DocumentName; if (!System.IO.File.Exists(physicalPath)) { CreateDirectory(fileBasePath); if (!System.IO.File.Exists(physicalPath)) { IOFileHelper.WriteFile(physicalPath, documentInformation.DocumentByte); } } filePath = Url.Content(fileBasePath + documentName); } } } if (string.IsNullOrEmpty(filePath) || !filePath.IsImage()) { filePath = Url.Content("~/assets/images/no-image.png"); documentName = "no-photo.jpg"; } if (string.IsNullOrEmpty(filePath) && type == 2) { filePath = Url.Content("~/assets/images/no-profile-image.png"); documentName = "no-profile-image.png"; } var result = new ImageResult(filePath, IOFileHelper.GetMIMEType(documentName)); return(result); }
/// <summary> /// 读取json文件获取贴吧标题列表 /// </summary> /// <param name="pagesize"></param> /// <param name="filePath"></param> /// <returns></returns> public async Task <bool> GetTitleListByJSON(string pagesize) { bool flag = false; var JsonContentPath = ConfigurationManager.AppSettings["JsonContentPath"];//E:\CrawlerNproData\jsonFileSave\ IOFileHelper iOFileHelper = new IOFileHelper(); var fileNames = iOFileHelper.GetFileNames(JsonContentPath); if (fileNames == null) { return(false); } foreach (var fileName in fileNames) { var file = fileName.ToString(); var result = await GetContextData(JsonContentPath + file + ".json", pagesize); flag = result; } return(flag); }
/// <summary> /// 搜索开始 /// </summary> /// <param name="url"></param> /// <param name="fileName"></param> public async void SearchStart(string url, string fileName, int type) { ViewEnum.searchEnum searchEmum = (ViewEnum.searchEnum)type; string JsonContentPath = ConfigurationManager.AppSettings["JsonContentPath"];//E:\CrawlerNproData\jsonFileSave\ Elements elements; string strGetConetex = null; CrHttpRequest crHttpRequest = new CrHttpRequest(); strGetConetex = await crHttpRequest.SentDataAsync(HttpMethod.Post, url); if (strGetConetex.Length > 0) { Regex regUrl = new Regex(@"(/p/[0-9]{10})"); Regex regTitle = new Regex(">(.*?)<"); //this.txtContent.Text = strGetConetex; Document document = NSoup.NSoupClient.Parse(strGetConetex); Elements elementTbNum = document.GetElementsByClass("card_infoNum"); Elements elementTbName = document.GetElementsByClass("card_title_fname"); Elements elementTbTitle = document.Select("li.j_thread_list").Select("li.clearfix"); List <TitleContentEntity> listContentEntity = new List <TitleContentEntity>(); //内容 //有错误 List <ResultEntity> listResultEntity = new List <ResultEntity>(); //所有内容 try { for (int i = 0; i < elementTbTitle.Count; i++) { var elementss = elementTbTitle[i]; Elements elementTbChildReplies = elementss.Select("span.threadlist_rep_num").Select("span.center_text"); Elements elementTbUrl = elementss.Select("a.j_th_tit"); Debug.WriteLine(elementTbChildReplies + "/n/r" + elementTbUrl); ResultEntity resultEntity = new ResultEntity(); if (elementTbUrl.ToString().Contains("href")) { var resultUrl = regUrl.Match(elementTbUrl.ToString()).ToString(); var tempData = regTitle.Match(elementTbUrl.ToString()).ToString(); var resultTitle = tempData.Substring(1, tempData.Length - 2); resultEntity.Title = resultTitle; resultEntity.Url = resultUrl; resultEntity.CreateCode = resultUrl.Substring(3, resultUrl.Length - 3);//校验码 } if (elementTbChildReplies.ToString().Contains("回复")) { Regex regReplies = new Regex(@">(.*?)<"); var tempData = regReplies.Match(elementTbChildReplies.ToString()).ToString(); var resultReplies = tempData.Substring(1, tempData.Length - 2); resultEntity.Replies = resultReplies; resultEntity.Visiable = 1;//默认显示1 } listResultEntity.Add(resultEntity); } CompareHelper compareHelper = new CompareHelper(); List <ResultEntity> filterListResultEntity = new List <ResultEntity>();//所有内容 var listkeyWorld = SelectKeyWorld(); if (listkeyWorld == null) { Debug.WriteLine("--------------获取关键字失败------------------"); } foreach (var item in listResultEntity) { if (compareHelper.Compare(item.Title.Trim(), listkeyWorld, "KeyWorld")) { filterListResultEntity.Add(item); } } IOFileHelper ioFileHelper = new IOFileHelper(); JsonHelper jsonHelper = new JsonHelper(); var jsonData = jsonHelper.SerializerJson(filterListResultEntity); if (jsonData == null) { Debug.WriteLine("写入json之前 序列化出错"); } /////写入json 文件之前 把实体转换为json字符串 //if (ioFileHelper.SaveJsonFile(JsonContentPath, "" + fileName + ".json", jsonData, false) == false) //{ // Debug.WriteLine("文件写入出错"); //} switch (searchEmum) { case ViewEnum.searchEnum.WriteMysql: TBTitleListService tBTitleListService = new TBTitleListService(); tBTitleListService.InsertTBTItleList(filterListResultEntity); //添加到mysql数据库// break; case ViewEnum.searchEnum.WriteSQLite: LocTitleListService locTitleListService = new LocTitleListService(); locTitleListService.InsertLocalTBTitleEx(filterListResultEntity); //添加到sqlite,去重update break; case ViewEnum.searchEnum.WriteAll: TBTitleListService tbTitleListService = new TBTitleListService(); tbTitleListService.InsertTBTItleList(filterListResultEntity); //添加到mysql数据库 LocTitleListService loctitleListService = new LocTitleListService(); loctitleListService.InsertLocalTBTitle(filterListResultEntity); break; } } catch (Exception e) { Debug.WriteLine("-------CrawlerNproView Error------" + e); } #region foreach // //********************************************************************************// // List<ContentEntity> listContentEntity = new List<ContentEntity>();//内容 有错误 应该是标题 // List<RepliesEneity> listRepliesEneity = new List<RepliesEneity>();//回复 // this.txtContent.Text = elementTbUrl.ToString(); // foreach (var itemPrplies in elementTbChildReplies) // { // RepliesEneity repliesEneity = new RepliesEneity(); // if (itemPrplies.ToString().Contains("回复")) // { // Regex regReplies = new Regex(@">(.*?)<"); // var tempData = regReplies.Match(itemPrplies.ToString()).ToString(); // var resultReplies = tempData.Substring(1, tempData.Length - 2); // repliesEneity.Replies = resultReplies; // listRepliesEneity.Add(repliesEneity); // } // } // foreach (var itemUrl in elementTbUrl) // { // ContentEntity contentEntity = new ContentEntity(); // if (itemUrl.ToString().Contains("href")) // { // // string reg = @"<a> *href=([""'])?(?(.*?)[^'""]+)\1[^>]*>"; // Regex regUrl = new Regex(@"(/p/[0-9]{10})"); // //<a href="/p/5388999580" title="求大佬帮忙p个图" target="_blank" class="j_th_tit ">求大佬帮忙p个图</a> // //Regex regUrl = new Regex(@"<a?* href=\/p\/^(.*?) */a> "); // var resultUrl = regUrl.Match(itemUrl.ToString()).ToString(); // Regex regTitle = new Regex(">(.*?)<"); // var tempData = regTitle.Match(itemUrl.ToString()).ToString(); // var resultTitle = tempData.Substring(1, tempData.Length - 2); // contentEntity.Title = resultTitle; // contentEntity.Url = resultUrl; // listContentEntity.Add(contentEntity); // } // } // List<ResultEntity> listResultEntity = new List<ResultEntity>();//所有的 // //TBTitleListService tBTitleListService = new TBTitleListService(); // // tBTitleListService.InsertTBTItleList(MySqlConn.GetMysqlConn(), listTitleListEntity);添加到数据库 // //Elements elementTbChildTitle = docomentChild.Select("a.j_th_tit"); // //this.txtContent.Text = elementTbChildReplies.ToString()+"//**//"+"/n/r"+ elementTbChildTitle.ToString() +"//***//"; #endregion } }