/// <summary>
/// 龙虎榜个股及明细
/// </summary>
public void GetStockLHB()
{
this.GetALLStockCode(); ///准备全部股票代码
Console.WriteLine("准备下载龙虎榜个股及明细页面");
var httpdownloader = new HTTP("gbk");
#region 准备下载队列
var queueUrl = new Queue <URL>(); ///key :url value 失败次数
var queueMxUrl = new Queue <URL>(); ///龙虎榜明细队列
foreach (var stock in this.stockCodeDict)
{
var urlgglhb = new URL(string.Format("http://data.10jqka.com.cn/market/lhbgg/code/{0}/", stock.Key), message: "个股龙虎榜");
queueUrl.Enqueue(urlgglhb);
}
#endregion
var excetionDict = new Dictionary <string, int>();
while (0 < queueUrl.Count) ///下载个股龙虎榜准备龙虎榜明细队列
{
var qItem = queueUrl.Dequeue();
var urlgglhb = qItem; ///个股龙虎榜URL
var stockCode = urlgglhb.Url.Substring(urlgglhb.Url.Length - 8).Replace("/", string.Empty); //http://data.10jqka.com.cn/market/lhbgg/code/600360/ //http://data.10jqka.com.cn/ifmarket/getnewlh/code/601228/date/2017-04-14/rid/7/
var refID = Guid.NewGuid().ToString().Replace("-", string.Empty);
var headers = new Dictionary <string, string>();
headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
headers.Add("Accept-Encoding", "gzip, deflate");
headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
headers.Add("Host", "data.10jqka.com.cn");
headers.Add("Referer", "http://data.10jqka.com.cn/market/longhu/");
headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36");
var pagegglhbHtml = httpdownloader.GetGzip(urlgglhb.Url, Encoding.GetEncoding("GBK"), headers); ///下载的个股龙虎榜页面
if (100 < pagegglhbHtml.Length) ///若下载的有数据
{
#region 分析个股龙虎榜明细
var tableStartIndex = pagegglhbHtml.IndexOf("<tbody>"); ///个股龙虎榜表格数据开始位置
var tableEndIndex = pagegglhbHtml.IndexOf("</tbody>"); ///个股龙虎榜表格数据结束位置
var tableHtml = pagegglhbHtml.Substring(tableStartIndex, tableEndIndex - tableStartIndex);
var tdArray = tableHtml.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries);
foreach (var td in tdArray)
{
if (td.Contains("rid") && td.Contains("date"))
{
var paramArray = td.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
var date = string.Empty;
var rid = string.Empty;
var code = string.Empty;
foreach (var param in paramArray)
{
if (param.Contains("code="))
{
code = param.Substring(6, 6);
}
else if (param.Contains("date="))
{
date = param.Substring(6, 10);
}
else if (param.Contains("rid="))
{
rid = param.Substring(5, 2).Replace("\"", string.Empty);
}
if (!string.IsNullOrWhiteSpace(code) && !string.IsNullOrWhiteSpace(date) && !string.IsNullOrWhiteSpace(rid))
{
var urlgglhbmx = string.Format("http://data.10jqka.com.cn/ifmarket/getnewlh/code/{0}/date/{1}/rid/{2}/", code, date, rid); ///个股龙虎榜明细
var mxUrl = new URL(urlgglhbmx);
mxUrl.Data["StockCode"] = code;
mxUrl.Data["Date"] = date;
mxUrl.Data["Rid"] = rid;
mxUrl.Data["ParentUrl"] = urlgglhb.Url;
mxUrl.Data["RefID"] = refID;
mxUrl.Message = "个股龙虎榜明细";
queueMxUrl.Enqueue(mxUrl);
}
}
}
}
#endregion
#region 个股龙虎榜数据保存
var gglhbData = new //个股龙虎榜 数据
{
StockCode = stockCode,
StockName = this.stockCodeDict[stockCode],
Url = urlgglhb,
CreateTime = DateTime.Now,
Page = pagegglhbHtml,
ContentType = "个股龙虎榜",
RefID = refID,///和明细保持一致
MD5 = Convertor.Encode_MD5(pagegglhbHtml),
TaskID = THS.CONST.TaskID
};
mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbData); ///个股龙虎榜
Console.WriteLine("个股龙虎榜 数据保存 {0} {1} {2}", stockCode, this.stockCodeDict[stockCode], queueUrl.Count);
#endregion
}
else
{
if (excetionDict.ContainsKey(qItem.Url) && 3 <= excetionDict[qItem.Url])
{
///写异常信息
Console.WriteLine("下载失败 {0}", qItem.Url);
var item = new Dictionary <string, object>();
item["ContentType"] = "龙虎榜下载失败异常信息";
item["TaskID"] = THS.CONST.TaskID;
item["CreateTime"] = DateTime.Now;
item["Data"] = Convertor.FromObjectToDictionary(qItem);
mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item);
Thread.Sleep(60 * 1000);
}
else
{
excetionDict[qItem.Url] = (excetionDict.ContainsKey(qItem.Url)) ? excetionDict[qItem.Url] + 1 : 0;
queueUrl.Enqueue(qItem);
}
}
}
var excetionMxDict = new Dictionary <string, int>();
while (0 < queueMxUrl.Count)
{
var qItem = queueMxUrl.Dequeue();
var headers = new Dictionary <string, string>();
headers.Add("Accept", "*/*");
headers.Add("Accept-Encoding", "gzip, deflate");
headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
headers.Add("Host", "data.10jqka.com.cn");
headers.Add("Referer", string.Format("http://data.10jqka.com.cn/market/lhbgg/code/{0}/", qItem.Data["StockCode"].ToString()));
headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36");
headers.Add("X-Requested-With", "XMLHttpRequest");
///下载明细
var subHtml = httpdownloader.GetGzip(qItem.Url, Encoding.GetEncoding("GBK"), headers);
if (50 < subHtml.Length)
{
var stockCode = qItem.Data["StockCode"].ToString();
var date = qItem.Data["Date"].ToString();
var rid = qItem.Data["Rid"].ToString();
var parentUrl = qItem.Data["ParentUrl"].ToString();
var refID = qItem.Data["RefID"].ToString();
var gglhbmxData = new
{
StockCode = qItem.Data["StockCode"].ToString(),
StockName = this.stockCodeDict[stockCode],
Date = date,
Rid = rid,
ParentUrl = parentUrl,
Url = qItem.Url,
CreateTime = DateTime.Now,
Page = subHtml,
ContentType = "个股龙虎榜明细",
RefID = refID,
MD5 = Convertor.Encode_MD5(subHtml),
TaskID = THS.CONST.TaskID
};
mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbmxData);
Console.WriteLine("个股龙虎榜明细 数据保存 {0} {1} {2}", stockCode, this.stockCodeDict[stockCode], queueMxUrl.Count);
///若能正常下载则删除
if (excetionMxDict.ContainsKey(qItem.Url))
{
///正常下载完毕删除异常信息
excetionMxDict.Remove(qItem.Url);
}
}
else
{
Console.WriteLine("下载异常 {0} ", qItem.Url);
if (excetionMxDict.ContainsKey(qItem.Url) && 3 <= excetionMxDict[qItem.Url])
{
///写异常信息
Console.WriteLine("下载失败 {0}", qItem.Url);
var item = new Dictionary <string, object>();
item["ContentType"] = "龙虎榜明细下载失败异常信息";
item["TaskID"] = THS.CONST.TaskID;
item["CreateTime"] = DateTime.Now;
item["Data"] = qItem;
mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item);
Thread.Sleep(60 * 1000);
}
else
{
///重新加入队列
excetionMxDict[qItem.Url] = (excetionMxDict.ContainsKey(qItem.Url)) ? excetionMxDict[qItem.Url] + 1 : 0;
queueMxUrl.Enqueue(qItem);
}
}
}
}
/// <summary>
/// 龙虎榜个股及明细
/// </summary>
public void GetStockLHB()
{
this.GetALLStockCode(); ///准备全部股票代码
Console.WriteLine("准备下载龙虎榜个股及明细页面");
var httpdownloader = new HTTP("gbk");
#region 准备下载队列
var queueUrl = new Queue <string>(); ///key :url value 失败次数
foreach (var stock in this.stockCodeDict)
{
var urlgglhb = string.Format("http://data.10jqka.com.cn/market/lhbgg/code/{0}/", stock.Key);//
queueUrl.Enqueue(urlgglhb);
}
#endregion
///若下载失败
httpdownloader.EventException += (object sender, EventArgs e) => {
var ee = e as EventProcEventArgs;
Console.WriteLine("下载失败 {0}", ee.Default);
var item = ee.Default as Dictionary <string, object>;
item["ContentType"] = "龙虎榜下载失败异常信息";
item["TaskID"] = THS.CONST.TaskID;
mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item);
queueUrl.Enqueue(item["Url"].ToString());
Thread.Sleep(60 * 1000);
};
Func <string, string, string, string, string, int> downloadLHBMX = (string code, string date, string rid, string urlgglhb, string refID) =>
{
var urlgglhbmx = string.Format("http://data.10jqka.com.cn/ifmarket/getnewlh/code/{0}/date/{1}/rid/{2}/", code, date, rid); ///个股龙虎榜明细
var subHtml = httpdownloader.GetGzip(urlgglhbmx);
var gglhbmxData = new
{
StockCode = code,
StockName = this.stockCodeDict[code],
Date = date,
Rid = rid,
ParentUrl = urlgglhb,
Url = urlgglhbmx,
CreateTime = DateTime.Now,
Page = subHtml,
ContentType = "个股龙虎榜明细",
RefID = refID,
MD5 = Convertor.Encode_MD5(subHtml),
TaskID = THS.CONST.TaskID
};
mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbmxData);
return(0);
};
while (0 < queueUrl.Count)
{
var qItem = queueUrl.Dequeue();
var urlgglhb = qItem; ///个股龙虎榜URL
var url = qItem;
var stockCode = urlgglhb.Substring(urlgglhb.Length - 8).Replace("/", string.Empty); //http://data.10jqka.com.cn/market/lhbgg/code/600360/ //http://data.10jqka.com.cn/ifmarket/getnewlh/code/601228/date/2017-04-14/rid/7/
var refID = Guid.NewGuid().ToString().Replace("-", string.Empty);
var pagegglhbHtml = httpdownloader.GetGzip(urlgglhb); ///下载的个股龙虎榜页面
if (100 < pagegglhbHtml.Length) ///若下载的有数据
{
#region 分析个股龙虎榜明细
var tableStartIndex = pagegglhbHtml.IndexOf("<tbody>"); ///个股龙虎榜表格数据开始位置
var tableEndIndex = pagegglhbHtml.IndexOf("</tbody>"); ///个股龙虎榜表格数据结束位置
var tableHtml = pagegglhbHtml.Substring(tableStartIndex, tableEndIndex - tableStartIndex);
var tdArray = tableHtml.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries);
foreach (var td in tdArray)
{
if (td.Contains("rid") && td.Contains("date"))
{
var paramArray = td.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
var date = string.Empty;
var rid = string.Empty;
var code = string.Empty;
foreach (var param in paramArray)
{
if (param.Contains("code="))
{
code = param.Substring(6, 6);
}
else if (param.Contains("date="))
{
date = param.Substring(6, 10);
}
else if (param.Contains("rid="))
{
rid = param.Substring(5, 2).Replace("\"", string.Empty);
}
if (!string.IsNullOrWhiteSpace(code) && !string.IsNullOrWhiteSpace(date) && !string.IsNullOrWhiteSpace(rid))
{
var urlgglhbmx = string.Format("http://data.10jqka.com.cn/ifmarket/getnewlh/code/{0}/date/{1}/rid/{2}/", code, date, rid); ///个股龙虎榜明细
var subHtml = httpdownloader.GetGzip(urlgglhbmx);
var gglhbmxData = new
{
StockCode = code,
StockName = this.stockCodeDict[code],
Date = date,
Rid = rid,
ParentUrl = urlgglhb,
Url = urlgglhbmx,
CreateTime = DateTime.Now,
Page = subHtml,
ContentType = "个股龙虎榜明细",
RefID = refID,
MD5 = Convertor.Encode_MD5(subHtml),
TaskID = THS.CONST.TaskID
};
mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbmxData);
}
}
}
}
#endregion
#region 数据保存
var gglhbData = new //个股龙虎榜 数据
{
StockCode = stockCode,
StockName = this.stockCodeDict[stockCode],
Url = urlgglhb,
CreateTime = DateTime.Now,
Page = pagegglhbHtml,
ContentType = "个股龙虎榜",
RefID = refID,
MD5 = Convertor.Encode_MD5(pagegglhbHtml),
TaskID = THS.CONST.TaskID
};
mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbData); ///个股龙虎榜
Console.WriteLine("个股龙虎榜 数据保存 {0} {1} {2}", stockCode, this.stockCodeDict[stockCode], queueUrl.Count);
#endregion
}
else
{
var item = new {
ContentType = "龙虎榜下载数据异常",
Page = pagegglhbHtml,
Url = urlgglhb,
CreateTime = DateTime.Now,
MD5 = Convertor.Encode_MD5(pagegglhbHtml),
TaskID = THS.CONST.TaskID
};
mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item);
}
}
}
