/// <summary>
/// Redis缓存依赖的文件路径
/// </summary>
/// <returns></returns>
private string GetRedisRelyFileDir()
{
AssemblyDataExt ass = new AssemblyDataExt();
string debugDir = ass.GetAssemblyDir();
return(debugDir + "/" + SystemConfig.RedisCacheFromFileReleative + "/" + typeof(CategoryGroup).Name + "/" + GetCagetoryDataFileNameOrRedisItem(targetCountry,
redisItemOrFileNameFormat(SystemConfig.RedisValueIsJsonFormat)));
}
static void QueryIdCard(string url)
{
string requestParam = @"Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
Accept-Encoding:gzip, deflate
Accept-Language:zh-CN,zh;q=0.8
Cache-Control:max-age=0
Connection:keep-alive
Cookie:ASPSESSIONIDSQABCDRA=AJCBIMLAFHDDPOKGJDPBGIHB; gxskydzx31_f5ee_saltkey=V8j84g4P; gxskydzx31_f5ee_lastvisit=1503227237; gxskydzx31_f5ee_sid=wL5R7g; gxskydzx31_f5ee_lastact=1503230837%09nav_js.php%09; UM_distinctid=15dff8a57c50-0ad597d1229d24-474a0521-15f900-15dff8a57c61; Card3=Url=; ASPSESSIONIDQQCACCRB=PBHHKPLACDPLIPHOAFPIJOBI; CNZZDATA1259785388=339484251-1503230374-%7C1503230374
Host:idcard.gxsky.com
Referer:{url}
Upgrade-Insecure-Requests:1
User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36";
requestParam = requestParam.Replace("{url}", url);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Headers.Add("Accept-Encoding", "gzip,deflate");
//request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
request.Method = "GET";
request.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:19.0) Gecko/20100101 Firefox/19.0";
request.KeepAlive = false;
request.AllowWriteStreamBuffering = true;//对响应数据进行缓冲处理
try
{
HttpWebResponse webResponse = null;
Stream str = null;
webResponse = (HttpWebResponse)request.GetResponse(); //此处有坑【如果http响应超时此处会抛出异常】
if (webResponse.StatusCode != HttpStatusCode.OK)
{ //http请求失败
}
str = webResponse.GetResponseStream();
GZipStream gzip = new GZipStream(str, CompressionMode.Decompress);//解压缩
Encoding enc = Encoding.GetEncoding("gb2312");
StreamReader sr = new StreamReader(gzip, enc);
string text = sr.ReadToEnd();
str.Dispose();
gzip.Close();
sr.Close();
// HttpClientExtend.GetHttpResponseData(url);
AssemblyDataExt ass = new AssemblyDataExt();
string dir = ass.ForeachDir(ass.GetAssemblyDir(), 3);
LoggerWriter.CreateLogFile(text, dir + "/" + ELogType.DataLog, ELogType.DataLog);
}
catch (Exception ex)
{
return;
}
}
static void QueryUserRemarkLossId()
{
int cur = 1;
string text = GetRequestData(string.Format(AppConfig.UserRemarkLoseIdCard, cur));
string startRegex = "<form method=post";
if (!text.Contains(startRegex))
{
return;
}
int position = text.IndexOf(startRegex);
string page = text.Substring(position + startRegex.Length);
string endRegex = "<input";
string releate = page.Substring(0, page.IndexOf(endRegex));
AssemblyDataExt ass = new AssemblyDataExt();
string dir = ass.ForeachDir(ass.GetAssemblyDir(), 3);
LoggerWriter.CreateLogFile(text, dir + "/" + ELogType.DataLog, ELogType.DataLog);
string sign = "#ff0000'>";
releate = releate.Substring(releate.LastIndexOf(sign) + sign.Length);
releate = releate.Substring(0, releate.LastIndexOf("</b>"));
Regex reg = new Regex("([0-9]+)");
MatchCollection list = reg.Matches(releate);
//--->"3500</font></b>条记录 <b>35"
Match total = list[0];// total
Match limit = list[1];
int sum = int.Parse(total.ToString());
int size = int.Parse(limit.ToString());
int pages = (sum / size) + (sum % size > 0 ? 1 : 0);
cur++;
while (cur <= pages)
{
text = GetRequestData(string.Format(AppConfig.UserRemarkLoseIdCard, cur));
cur++;
if (string.IsNullOrEmpty(text))
{
continue;
}
LoggerWriter.CreateLogFile(text, dir + "/" + ELogType.DataLog, ELogType.DataLog);
}
}
/// <summary>
/// 对于从数据库中读取的城市数据进行处理写入到文本文件中作为基础数据使用
/// </summary>
/// <param name="data"></param>
/// <param name="root"></param>
private void AnalyCity(List <CategoryData> data, CategoryData root)
{
AssemblyDataExt ass = new AssemblyDataExt();
string debugDir = ass.GetAssemblyDir() + "/" + SystemConfig.RedisCacheFromFileReleative;
CategoryGourpHelper helper = new CategoryGourpHelper();
CategoryGroup result = helper.DataGroup(data);;
CategoryGroup nodes = new CategoryGroup();// (CategoryGroup)result;//提取国家列表
string cityDir = debugDir + "/" + typeof(CategoryGroup).Name;
//中国的省会列表
foreach (CategoryGroup item in result.Childrens)
{//查询到国家，过滤省市直辖区
//item.Childrens = new List<CategoryGroup>();
CategoryGroup temp = new CategoryGroup()
{
Root = item.Root
};
nodes.Childrens.Add(temp);
}
string country = nodes.ConvertJson();
Logger.CreateNewAppData(country, "Country.txt", cityDir);
//提取中国的省会列表
CategoryGroup provinceGroup = new CategoryGroup();
CategoryGroup china = result.Childrens.Where(s => s.Root.Name == root.Name).FirstOrDefault();
provinceGroup.Root = china.Root;
foreach (CategoryGroup item in china.Childrens)
{//查询省会，过滤到市、区
provinceGroup.Childrens.Add(new CategoryGroup()
{
Root = item.Root
});
//item.Childrens = new List<CategoryGroup>();
}
string province = provinceGroup.ConvertJson();
Logger.CreateNewAppData(province, GetNodeItemFileName(china,
redisItemOrFileNameFormat(SystemConfig.RedisValueIsJsonFormat)),
GetRedisRelyFileDir());
CategoryGroup cgJson = province.ConvertObject <CategoryGroup>();
//各个省会的市区列表
foreach (var item in china.Childrens)
{
string path = LogPrepare.GetLogPath();
LoggerWriter.CreateLogFile(item.Root.Name, path,
ELogType.DebugData, DateTime.Now.ToString(Common.Data.CommonFormat.DateToHourIntFormat) + ".log", true);
//提取省会列表到市
CategoryGroup level = new CategoryGroup()
{
Childrens = item.Childrens.Select(s => new CategoryGroup()
{
Root = s.Root
}).ToList()
};
string jsonNode = level.ConvertJson();
//string provinceJson=item.ConvertJson();
if (!string.IsNullOrEmpty(jsonNode))
{
Logger.CreateNewAppData(jsonNode, GetNodeItemFileName(item,
redisItemOrFileNameFormat(SystemConfig.RedisValueIsJsonFormat)),
GetRedisRelyFileDir());
if (SystemConfig.OpenRedis)
{
}
}
CategoryGroup cg = new CategoryGroup()
{
Root = item.Root,
Childrens = item.Childrens
};
LoggerWriter.CreateLogFile(cg.Root.Name, path,
ELogType.DebugData, DateTime.Now.ToString(Common.Data.CommonFormat.DateToHourIntFormat) + ".log", true);
foreach (CategoryGroup c in item.Childrens)
{//省直辖市的子节点
CategoryGroup cn = new CategoryGroup()
{
Root = c.Root,
Childrens = c.Childrens
};
string nodeJson = string.Empty;
if (c.Childrens.Count > 0)
{
nodeJson = cn.ConvertJson();
}
LoggerWriter.CreateLogFile(cn.Root.Name, path,
ELogType.DebugData, DateTime.Now.ToString(Common.Data.CommonFormat.DateToHourIntFormat) + ".log", true);
Logger.CreateNewAppData(nodeJson, GetNodeItemFileName(c,
redisItemOrFileNameFormat(SystemConfig.RedisValueIsJsonFormat)), GetRedisRelyFileDir());
}
}
}
