/// <summary>
/// get keyworld content from text file
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public List <KeyWorldEntity> getTextContent(string url)
{
List <KeyWorldEntity> listkeyWorldEntity = new List <KeyWorldEntity>();
try
{
if (string.IsNullOrEmpty(url))
{
return(null);
}
IOFileHelper iOFileHelper = new IOFileHelper();
var strContext = iOFileHelper.ReadTxtFile(url);
var byteData = strContext.Split('-');
foreach (var item in byteData)
{
KeyWorldEntity keyWorldEntity = new KeyWorldEntity();
keyWorldEntity.KeyWorld = item.Trim();
listkeyWorldEntity.Add(keyWorldEntity);
}
return(listkeyWorldEntity);
}
catch
{
Debug.WriteLine("KeyWorldService");
}
return(listkeyWorldEntity);
}
/// <summary>
/// read json file to list
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public List <BarNameEntity> getJsonContent(string filepath)
{
List <BarNameEntity> listkeyWorldEntity = new List <BarNameEntity>();
try
{
if (string.IsNullOrEmpty(filepath))
{
return(null);
}
IOFileHelper iOFileHelper = new IOFileHelper();
var strContext = iOFileHelper.ReadTxtFile(filepath);
var byteData = strContext.Split('-');
UrlCodeConvert urlCodeConvert = new UrlCodeConvert();
foreach (var item in byteData)
{
BarNameEntity barNameEntity = new BarNameEntity();
barNameEntity.BarName = item.Trim();
barNameEntity.Url = urlCodeConvert.EncodeUrl(item.Trim());
listkeyWorldEntity.Add(barNameEntity);
}
return(listkeyWorldEntity);
}
catch
{
Debug.WriteLine("KeyWorldService");
}
return(listkeyWorldEntity);
}
/// <summary>
/// 读取applocal json 文件贴吧Name
/// </summary>
/// <param name="JsonPath"></param>
public static List <SearchlistEntity> ReadTBNameFile(Uri JsonPath = null)
{
try
{
List <SearchlistEntity> listSearchlistEntity = new List <SearchlistEntity>();
IOFileHelper iOFileHelper = new IOFileHelper();
var strJson = iOFileHelper.ReadJsonFileToString(@"/CrawlerNproData/BarName.json");
JsonHelper jsonHelper = new JsonHelper();
var listEntity = jsonHelper.DeserializeJsonTo <List <BarNameEntity> >(strJson);
UrlCodeConvert ucc = new UrlCodeConvert();
foreach (var item in listEntity)
{
SearchlistEntity searchlistEntity = new SearchlistEntity();
searchlistEntity.Title = item.BarName;
searchlistEntity.SearchContent = ucc.DecodeUrl(item.Url);
listSearchlistEntity.Add(searchlistEntity);
}
return(listSearchlistEntity);
}
catch
{
Debug.WriteLine("-------cls-CrawlerNproView------func ReadTBNameFile----");
}
return(null);
}
///java sys
/// <summary>
/// read database to local json file
/// </summary>
public void SaveTBNameToJson(Uri JsonPath = null)
{
//读取数据库
SysTBNameDAL sysTBNAMEDAL = new SysTBNameDAL();
var listTBName = sysTBNAMEDAL.SelectTBName(null);//token
//转换json
JsonHelper jsonHelper = new JsonHelper();
var strJson = jsonHelper.SerializerJson(listTBName);
//保存文件
IOFileHelper iOFileHelper = new IOFileHelper();
string JsonContentPath = ConfigurationManager.AppSettings["JsonContentPath"];
iOFileHelper.SaveJsonFile(JsonContentPath, "BarName.json", strJson);//文件名写死
}
public ActionResult ShowImage(int globalId, string documentName, int type = 1)
{
const string fileBasePath = Constants.ImagePath.ImageFolderPath;
string filePath = string.Empty;
if (!string.IsNullOrEmpty(documentName) && globalId > 0)
{
string physicalPath = System.Web.HttpContext.Current.Server.MapPath(fileBasePath) + documentName;
if (System.IO.File.Exists(physicalPath))
{
filePath = Url.Content(fileBasePath + documentName);
}
else
{
var service = DependencyResolver.Current.GetService(typeof(IDocumentInformationService)) as IDocumentInformationService;
var documentInformation = service.Get(new DocumentInformation()
{
GlobalId = globalId
});
if (documentInformation != null && !string.IsNullOrEmpty(documentInformation.DocumentName))
{
documentName = documentInformation.DocumentName;
if (!System.IO.File.Exists(physicalPath))
{
CreateDirectory(fileBasePath);
if (!System.IO.File.Exists(physicalPath))
{
IOFileHelper.WriteFile(physicalPath, documentInformation.DocumentByte);
}
}
filePath = Url.Content(fileBasePath + documentName);
}
}
}
if (string.IsNullOrEmpty(filePath) || !filePath.IsImage())
{
filePath = Url.Content("~/assets/images/no-image.png");
documentName = "no-photo.jpg";
}
if (string.IsNullOrEmpty(filePath) && type == 2)
{
filePath = Url.Content("~/assets/images/no-profile-image.png");
documentName = "no-profile-image.png";
}
var result = new ImageResult(filePath, IOFileHelper.GetMIMEType(documentName));
return(result);
}
/// <summary>
/// 读取json文件获取贴吧标题列表
/// </summary>
/// <param name="pagesize"></param>
/// <param name="filePath"></param>
/// <returns></returns>
public async Task <bool> GetTitleListByJSON(string pagesize)
{
bool flag = false;
var JsonContentPath = ConfigurationManager.AppSettings["JsonContentPath"];//E:\CrawlerNproData\jsonFileSave\
IOFileHelper iOFileHelper = new IOFileHelper();
var fileNames = iOFileHelper.GetFileNames(JsonContentPath);
if (fileNames == null)
{
return(false);
}
foreach (var fileName in fileNames)
{
var file = fileName.ToString();
var result = await GetContextData(JsonContentPath + file + ".json", pagesize);
flag = result;
}
return(flag);
}
/// <summary>
/// 搜索开始
/// </summary>
/// <param name="url"></param>
/// <param name="fileName"></param>
public async void SearchStart(string url, string fileName, int type)
{
ViewEnum.searchEnum searchEmum = (ViewEnum.searchEnum)type;
string JsonContentPath = ConfigurationManager.AppSettings["JsonContentPath"];//E:\CrawlerNproData\jsonFileSave\
Elements elements;
string strGetConetex = null;
CrHttpRequest crHttpRequest = new CrHttpRequest();
strGetConetex = await crHttpRequest.SentDataAsync(HttpMethod.Post, url);
if (strGetConetex.Length > 0)
{
Regex regUrl = new Regex(@"(/p/[0-9]{10})");
Regex regTitle = new Regex(">(.*?)<");
//this.txtContent.Text = strGetConetex;
Document document = NSoup.NSoupClient.Parse(strGetConetex);
Elements elementTbNum = document.GetElementsByClass("card_infoNum");
Elements elementTbName = document.GetElementsByClass("card_title_fname");
Elements elementTbTitle = document.Select("li.j_thread_list").Select("li.clearfix");
List <TitleContentEntity> listContentEntity = new List <TitleContentEntity>(); //内容 //有错误
List <ResultEntity> listResultEntity = new List <ResultEntity>(); //所有内容
try
{
for (int i = 0; i < elementTbTitle.Count; i++)
{
var elementss = elementTbTitle[i];
Elements elementTbChildReplies = elementss.Select("span.threadlist_rep_num").Select("span.center_text");
Elements elementTbUrl = elementss.Select("a.j_th_tit");
Debug.WriteLine(elementTbChildReplies + "/n/r" + elementTbUrl);
ResultEntity resultEntity = new ResultEntity();
if (elementTbUrl.ToString().Contains("href"))
{
var resultUrl = regUrl.Match(elementTbUrl.ToString()).ToString();
var tempData = regTitle.Match(elementTbUrl.ToString()).ToString();
var resultTitle = tempData.Substring(1, tempData.Length - 2);
resultEntity.Title = resultTitle;
resultEntity.Url = resultUrl;
resultEntity.CreateCode = resultUrl.Substring(3, resultUrl.Length - 3);//校验码
}
if (elementTbChildReplies.ToString().Contains("回复"))
{
Regex regReplies = new Regex(@">(.*?)<");
var tempData = regReplies.Match(elementTbChildReplies.ToString()).ToString();
var resultReplies = tempData.Substring(1, tempData.Length - 2);
resultEntity.Replies = resultReplies;
resultEntity.Visiable = 1;//默认显示1
}
listResultEntity.Add(resultEntity);
}
CompareHelper compareHelper = new CompareHelper();
List <ResultEntity> filterListResultEntity = new List <ResultEntity>();//所有内容
var listkeyWorld = SelectKeyWorld();
if (listkeyWorld == null)
{
Debug.WriteLine("--------------获取关键字失败------------------");
}
foreach (var item in listResultEntity)
{
if (compareHelper.Compare(item.Title.Trim(), listkeyWorld, "KeyWorld"))
{
filterListResultEntity.Add(item);
}
}
IOFileHelper ioFileHelper = new IOFileHelper();
JsonHelper jsonHelper = new JsonHelper();
var jsonData = jsonHelper.SerializerJson(filterListResultEntity);
if (jsonData == null)
{
Debug.WriteLine("写入json之前 序列化出错");
}
/////写入json 文件之前 把实体转换为json字符串
//if (ioFileHelper.SaveJsonFile(JsonContentPath, "" + fileName + ".json", jsonData, false) == false)
//{
// Debug.WriteLine("文件写入出错");
//}
switch (searchEmum)
{
case ViewEnum.searchEnum.WriteMysql:
TBTitleListService tBTitleListService = new TBTitleListService();
tBTitleListService.InsertTBTItleList(filterListResultEntity); //添加到mysql数据库//
break;
case ViewEnum.searchEnum.WriteSQLite:
LocTitleListService locTitleListService = new LocTitleListService();
locTitleListService.InsertLocalTBTitleEx(filterListResultEntity); //添加到sqlite，去重update
break;
case ViewEnum.searchEnum.WriteAll:
TBTitleListService tbTitleListService = new TBTitleListService();
tbTitleListService.InsertTBTItleList(filterListResultEntity); //添加到mysql数据库
LocTitleListService loctitleListService = new LocTitleListService();
loctitleListService.InsertLocalTBTitle(filterListResultEntity);
break;
}
}
catch (Exception e)
{
Debug.WriteLine("-------CrawlerNproView Error------" + e);
}
#region foreach
// //********************************************************************************//
// List<ContentEntity> listContentEntity = new List<ContentEntity>();//内容 有错误 应该是标题
// List<RepliesEneity> listRepliesEneity = new List<RepliesEneity>();//回复
// this.txtContent.Text = elementTbUrl.ToString();
// foreach (var itemPrplies in elementTbChildReplies)
// {
// RepliesEneity repliesEneity = new RepliesEneity();
// if (itemPrplies.ToString().Contains("回复"))
// {
// Regex regReplies = new Regex(@">(.*?)<");
// var tempData = regReplies.Match(itemPrplies.ToString()).ToString();
// var resultReplies = tempData.Substring(1, tempData.Length - 2);
// repliesEneity.Replies = resultReplies;
// listRepliesEneity.Add(repliesEneity);
// }
// }
// foreach (var itemUrl in elementTbUrl)
// {
// ContentEntity contentEntity = new ContentEntity();
// if (itemUrl.ToString().Contains("href"))
// {
// // string reg = @"<a> *href=([""'])?(?(.*?)[^'""]+)\1[^>]*>";
// Regex regUrl = new Regex(@"(/p/[0-9]{10})");
// //<a href="/p/5388999580" title="求大佬帮忙p个图" target="_blank" class="j_th_tit ">求大佬帮忙p个图</a>
// //Regex regUrl = new Regex(@"<a?* href=\/p\/^(.*?) */a> ");
// var resultUrl = regUrl.Match(itemUrl.ToString()).ToString();
// Regex regTitle = new Regex(">(.*?)<");
// var tempData = regTitle.Match(itemUrl.ToString()).ToString();
// var resultTitle = tempData.Substring(1, tempData.Length - 2);
// contentEntity.Title = resultTitle;
// contentEntity.Url = resultUrl;
// listContentEntity.Add(contentEntity);
// }
// }
// List<ResultEntity> listResultEntity = new List<ResultEntity>();//所有的
// //TBTitleListService tBTitleListService = new TBTitleListService();
// // tBTitleListService.InsertTBTItleList(MySqlConn.GetMysqlConn(), listTitleListEntity);添加到数据库
// //Elements elementTbChildTitle = docomentChild.Select("a.j_th_tit");
// //this.txtContent.Text = elementTbChildReplies.ToString()+"//**//"+"/n/r"+ elementTbChildTitle.ToString() +"//***//";
#endregion
}
}
