/// <summary> /// 龙虎榜个股及明细 /// </summary> public void GetStockLHB() { this.GetALLStockCode(); ///准备全部股票代码 Console.WriteLine("准备下载龙虎榜个股及明细页面"); var httpdownloader = new HTTP("gbk"); #region 准备下载队列 var queueUrl = new Queue <URL>(); ///key :url value 失败次数 var queueMxUrl = new Queue <URL>(); ///龙虎榜明细队列 foreach (var stock in this.stockCodeDict) { var urlgglhb = new URL(string.Format("http://data.10jqka.com.cn/market/lhbgg/code/{0}/", stock.Key), message: "个股龙虎榜"); queueUrl.Enqueue(urlgglhb); } #endregion var excetionDict = new Dictionary <string, int>(); while (0 < queueUrl.Count) ///下载个股龙虎榜准备龙虎榜明细队列 { var qItem = queueUrl.Dequeue(); var urlgglhb = qItem; ///个股龙虎榜URL var stockCode = urlgglhb.Url.Substring(urlgglhb.Url.Length - 8).Replace("/", string.Empty); //http://data.10jqka.com.cn/market/lhbgg/code/600360/ //http://data.10jqka.com.cn/ifmarket/getnewlh/code/601228/date/2017-04-14/rid/7/ var refID = Guid.NewGuid().ToString().Replace("-", string.Empty); var headers = new Dictionary <string, string>(); headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); headers.Add("Accept-Encoding", "gzip, deflate"); headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"); headers.Add("Host", "data.10jqka.com.cn"); headers.Add("Referer", "http://data.10jqka.com.cn/market/longhu/"); headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); var pagegglhbHtml = httpdownloader.GetGzip(urlgglhb.Url, Encoding.GetEncoding("GBK"), headers); ///下载的个股龙虎榜页面 if (100 < pagegglhbHtml.Length) ///若下载的有数据 { #region 分析个股龙虎榜明细 var tableStartIndex = pagegglhbHtml.IndexOf("<tbody>"); ///个股龙虎榜表格数据开始位置 var tableEndIndex = pagegglhbHtml.IndexOf("</tbody>"); ///个股龙虎榜表格数据结束位置 var tableHtml = pagegglhbHtml.Substring(tableStartIndex, tableEndIndex - tableStartIndex); var tdArray = tableHtml.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries); foreach (var td in tdArray) { if (td.Contains("rid") && td.Contains("date")) { var paramArray = td.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries); var date = string.Empty; var rid = string.Empty; var code = string.Empty; foreach (var param in paramArray) { if (param.Contains("code=")) { code = param.Substring(6, 6); } else if (param.Contains("date=")) { date = param.Substring(6, 10); } else if (param.Contains("rid=")) { rid = param.Substring(5, 2).Replace("\"", string.Empty); } if (!string.IsNullOrWhiteSpace(code) && !string.IsNullOrWhiteSpace(date) && !string.IsNullOrWhiteSpace(rid)) { var urlgglhbmx = string.Format("http://data.10jqka.com.cn/ifmarket/getnewlh/code/{0}/date/{1}/rid/{2}/", code, date, rid); ///个股龙虎榜明细 var mxUrl = new URL(urlgglhbmx); mxUrl.Data["StockCode"] = code; mxUrl.Data["Date"] = date; mxUrl.Data["Rid"] = rid; mxUrl.Data["ParentUrl"] = urlgglhb.Url; mxUrl.Data["RefID"] = refID; mxUrl.Message = "个股龙虎榜明细"; queueMxUrl.Enqueue(mxUrl); } } } } #endregion #region 个股龙虎榜数据保存 var gglhbData = new //个股龙虎榜 数据 { StockCode = stockCode, StockName = this.stockCodeDict[stockCode], Url = urlgglhb, CreateTime = DateTime.Now, Page = pagegglhbHtml, ContentType = "个股龙虎榜", RefID = refID,///和明细保持一致 MD5 = Convertor.Encode_MD5(pagegglhbHtml), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbData); ///个股龙虎榜 Console.WriteLine("个股龙虎榜 数据保存 {0} {1} {2}", stockCode, this.stockCodeDict[stockCode], queueUrl.Count); #endregion } else { if (excetionDict.ContainsKey(qItem.Url) && 3 <= excetionDict[qItem.Url]) { ///写异常信息 Console.WriteLine("下载失败 {0}", qItem.Url); var item = new Dictionary <string, object>(); item["ContentType"] = "龙虎榜下载失败异常信息"; item["TaskID"] = THS.CONST.TaskID; item["CreateTime"] = DateTime.Now; item["Data"] = Convertor.FromObjectToDictionary(qItem); mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item); Thread.Sleep(60 * 1000); } else { excetionDict[qItem.Url] = (excetionDict.ContainsKey(qItem.Url)) ? excetionDict[qItem.Url] + 1 : 0; queueUrl.Enqueue(qItem); } } } var excetionMxDict = new Dictionary <string, int>(); while (0 < queueMxUrl.Count) { var qItem = queueMxUrl.Dequeue(); var headers = new Dictionary <string, string>(); headers.Add("Accept", "*/*"); headers.Add("Accept-Encoding", "gzip, deflate"); headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"); headers.Add("Host", "data.10jqka.com.cn"); headers.Add("Referer", string.Format("http://data.10jqka.com.cn/market/lhbgg/code/{0}/", qItem.Data["StockCode"].ToString())); headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); headers.Add("X-Requested-With", "XMLHttpRequest"); ///下载明细 var subHtml = httpdownloader.GetGzip(qItem.Url, Encoding.GetEncoding("GBK"), headers); if (50 < subHtml.Length) { var stockCode = qItem.Data["StockCode"].ToString(); var date = qItem.Data["Date"].ToString(); var rid = qItem.Data["Rid"].ToString(); var parentUrl = qItem.Data["ParentUrl"].ToString(); var refID = qItem.Data["RefID"].ToString(); var gglhbmxData = new { StockCode = qItem.Data["StockCode"].ToString(), StockName = this.stockCodeDict[stockCode], Date = date, Rid = rid, ParentUrl = parentUrl, Url = qItem.Url, CreateTime = DateTime.Now, Page = subHtml, ContentType = "个股龙虎榜明细", RefID = refID, MD5 = Convertor.Encode_MD5(subHtml), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbmxData); Console.WriteLine("个股龙虎榜明细 数据保存 {0} {1} {2}", stockCode, this.stockCodeDict[stockCode], queueMxUrl.Count); ///若能正常下载则删除 if (excetionMxDict.ContainsKey(qItem.Url)) { ///正常下载完毕删除异常信息 excetionMxDict.Remove(qItem.Url); } } else { Console.WriteLine("下载异常 {0} ", qItem.Url); if (excetionMxDict.ContainsKey(qItem.Url) && 3 <= excetionMxDict[qItem.Url]) { ///写异常信息 Console.WriteLine("下载失败 {0}", qItem.Url); var item = new Dictionary <string, object>(); item["ContentType"] = "龙虎榜明细下载失败异常信息"; item["TaskID"] = THS.CONST.TaskID; item["CreateTime"] = DateTime.Now; item["Data"] = qItem; mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item); Thread.Sleep(60 * 1000); } else { ///重新加入队列 excetionMxDict[qItem.Url] = (excetionMxDict.ContainsKey(qItem.Url)) ? excetionMxDict[qItem.Url] + 1 : 0; queueMxUrl.Enqueue(qItem); } } } }
/// <summary> /// 龙虎榜个股及明细 /// </summary> public void GetStockLHB() { this.GetALLStockCode(); ///准备全部股票代码 Console.WriteLine("准备下载龙虎榜个股及明细页面"); var httpdownloader = new HTTP("gbk"); #region 准备下载队列 var queueUrl = new Queue <string>(); ///key :url value 失败次数 foreach (var stock in this.stockCodeDict) { var urlgglhb = string.Format("http://data.10jqka.com.cn/market/lhbgg/code/{0}/", stock.Key);// queueUrl.Enqueue(urlgglhb); } #endregion ///若下载失败 httpdownloader.EventException += (object sender, EventArgs e) => { var ee = e as EventProcEventArgs; Console.WriteLine("下载失败 {0}", ee.Default); var item = ee.Default as Dictionary <string, object>; item["ContentType"] = "龙虎榜下载失败异常信息"; item["TaskID"] = THS.CONST.TaskID; mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item); queueUrl.Enqueue(item["Url"].ToString()); Thread.Sleep(60 * 1000); }; Func <string, string, string, string, string, int> downloadLHBMX = (string code, string date, string rid, string urlgglhb, string refID) => { var urlgglhbmx = string.Format("http://data.10jqka.com.cn/ifmarket/getnewlh/code/{0}/date/{1}/rid/{2}/", code, date, rid); ///个股龙虎榜明细 var subHtml = httpdownloader.GetGzip(urlgglhbmx); var gglhbmxData = new { StockCode = code, StockName = this.stockCodeDict[code], Date = date, Rid = rid, ParentUrl = urlgglhb, Url = urlgglhbmx, CreateTime = DateTime.Now, Page = subHtml, ContentType = "个股龙虎榜明细", RefID = refID, MD5 = Convertor.Encode_MD5(subHtml), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbmxData); return(0); }; while (0 < queueUrl.Count) { var qItem = queueUrl.Dequeue(); var urlgglhb = qItem; ///个股龙虎榜URL var url = qItem; var stockCode = urlgglhb.Substring(urlgglhb.Length - 8).Replace("/", string.Empty); //http://data.10jqka.com.cn/market/lhbgg/code/600360/ //http://data.10jqka.com.cn/ifmarket/getnewlh/code/601228/date/2017-04-14/rid/7/ var refID = Guid.NewGuid().ToString().Replace("-", string.Empty); var pagegglhbHtml = httpdownloader.GetGzip(urlgglhb); ///下载的个股龙虎榜页面 if (100 < pagegglhbHtml.Length) ///若下载的有数据 { #region 分析个股龙虎榜明细 var tableStartIndex = pagegglhbHtml.IndexOf("<tbody>"); ///个股龙虎榜表格数据开始位置 var tableEndIndex = pagegglhbHtml.IndexOf("</tbody>"); ///个股龙虎榜表格数据结束位置 var tableHtml = pagegglhbHtml.Substring(tableStartIndex, tableEndIndex - tableStartIndex); var tdArray = tableHtml.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries); foreach (var td in tdArray) { if (td.Contains("rid") && td.Contains("date")) { var paramArray = td.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries); var date = string.Empty; var rid = string.Empty; var code = string.Empty; foreach (var param in paramArray) { if (param.Contains("code=")) { code = param.Substring(6, 6); } else if (param.Contains("date=")) { date = param.Substring(6, 10); } else if (param.Contains("rid=")) { rid = param.Substring(5, 2).Replace("\"", string.Empty); } if (!string.IsNullOrWhiteSpace(code) && !string.IsNullOrWhiteSpace(date) && !string.IsNullOrWhiteSpace(rid)) { var urlgglhbmx = string.Format("http://data.10jqka.com.cn/ifmarket/getnewlh/code/{0}/date/{1}/rid/{2}/", code, date, rid); ///个股龙虎榜明细 var subHtml = httpdownloader.GetGzip(urlgglhbmx); var gglhbmxData = new { StockCode = code, StockName = this.stockCodeDict[code], Date = date, Rid = rid, ParentUrl = urlgglhb, Url = urlgglhbmx, CreateTime = DateTime.Now, Page = subHtml, ContentType = "个股龙虎榜明细", RefID = refID, MD5 = Convertor.Encode_MD5(subHtml), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbmxData); } } } } #endregion #region 数据保存 var gglhbData = new //个股龙虎榜 数据 { StockCode = stockCode, StockName = this.stockCodeDict[stockCode], Url = urlgglhb, CreateTime = DateTime.Now, Page = pagegglhbHtml, ContentType = "个股龙虎榜", RefID = refID, MD5 = Convertor.Encode_MD5(pagegglhbHtml), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbData); ///个股龙虎榜 Console.WriteLine("个股龙虎榜 数据保存 {0} {1} {2}", stockCode, this.stockCodeDict[stockCode], queueUrl.Count); #endregion } else { var item = new { ContentType = "龙虎榜下载数据异常", Page = pagegglhbHtml, Url = urlgglhb, CreateTime = DateTime.Now, MD5 = Convertor.Encode_MD5(pagegglhbHtml), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item); } } }