/// <summary> /// 从网页获取所有的股票代码 /// </summary> public Dictionary <string, string> GetAllStockCode() { var stockCodeDict = new Dictionary <string, string>(); var httpdownloader = new HTTP(); var headers = new Dictionary <string, string>(); headers.Add("Accept", "text/html,*/*; q=0.01"); headers.Add("Accept-Encoding", "gzip, deflate"); headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"); headers.Add("Host", "q.10jqka.com.cn"); headers.Add("Referer", "http://q.10jqka.com.cn/"); headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); headers.Add("X-Requested-With", "XMLHttpRequest"); #region 获取分页数 var pageCount = 0; var pagerUrl = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", 1); var pagerHtml = httpdownloader.GetGzip(pagerUrl, Encoding.GetEncoding("GBK"), headers); if (100 < pagerHtml.Length) { pageCount = int.Parse(pagerHtml.Substring(pagerHtml.LastIndexOf("</span>") - 3, 3)); ///页码数 Console.WriteLine("股票代码页数{0}\t{1}", pageCount, DateTime.Now); } #endregion var stringBuilder = new StringBuilder(); for (int i = 1; i <= pageCount; i++) { var url = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", i); var html = httpdownloader.GetGzip(url, Encoding.GetEncoding("GBK"), headers); #region 股票代码处理 string[] trArray = html.Substring(html.IndexOf("<tbody>") + "<tbody>".Length).Split(new string[] { "<tr>", "</tr>" }, StringSplitOptions.RemoveEmptyEntries); foreach (var tr in trArray) { var tdArray = tr.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries); if (10 < tdArray.Length) { var stockCode = tdArray[3].Substring(tdArray[3].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", ""); var stockName = tdArray[5].Substring(tdArray[5].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", ""); if (!stockCodeDict.ContainsKey(stockCode) && !stockCodeDict.ContainsValue(stockName)) { stockCodeDict.Add(stockCode, stockName); } else { throw new Exception("数据重复"); } Console.WriteLine("正在添加 {0}\t{1} 当前第{2}页 共{3}页", stockCode, stockName, i + 1, pageCount); } } #endregion Thread.Sleep(new Random().Next(1000, 3000)); } return(stockCodeDict); }
public void GetPageDaDa() { while (true) { if ((DateTime.Today.AddHours(16) < DateTime.Now) &&///每日收盘后 !(DateTime.Now.DayOfWeek == DayOfWeek.Sunday || DateTime.Now.DayOfWeek == DayOfWeek.Saturday)) { #region var httpDownloader = new HTTP("GBK"); var headers = new Dictionary <string, string>(); headers.Add("Accept", "*/*"); headers.Add("Accept-Encoding", "gzip,deflate"); headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"); headers.Add("Content-type", "application/x-www-form-urlencoded"); headers.Add("Host", "vip.stock.finance.sina.com.cn"); headers.Add("Referer", "http://vip.stock.finance.sina.com.cn/quotes_service/view/cn_bill_all.php"); headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); var volume = 100; ///一手 string pageCountUrl = string.Format("http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/CN_Bill.GetBillListCount?num=100&page=1363&sort=ticktime&asc=0&volume={0}&type=0", volume); //(new String("283802")) string pageCountText = httpDownloader.GetGzip(pageCountUrl, Encoding.GetEncoding("GBK"), headers); var pageCount = 0; var pageSize = 100; if (pageCountText.Contains("(new String(")) { pageCountText = pageCountText.Replace("(new String(", string.Empty).Replace(")", string.Empty).Replace("\"", string.Empty).Trim('\0'); pageCount = (0 < int.Parse(pageCountText) % volume) ? int.Parse(pageCountText) / pageSize + 1 : int.Parse(pageCountText) / pageSize; } for (int k = 1; k <= pageCount; k++) { var url = string.Format("http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/CN_Bill.GetBillList?num=100&page={0}&sort=ticktime&asc=0&volume={1}&type=0", k, volume); var pageContent = httpDownloader.GetGzip(url, Encoding.GetEncoding("GBK"), headers); var data = new { ContentType = "SINA大单追踪实时数据", Url = url, CreatTime = DateTime.Now, Page = pageContent, MD5 = Convertor.Encode_MD5(pageContent), TaskID = THS.CONST.TaskID, TradingDate = DateTime.Now }; mongo.Save("PageSource", "PageDaDan", data); Console.WriteLine("正在下载 {0} 共 {1}页 ", url, pageCount); Thread.Sleep(1000); } #endregion } else { Console.WriteLine("资金流向大单追踪 目前没在开市时间 工作日 9:30 - 15:00 当前时间 " + DateTime.Now); Thread.Sleep(60 * 1000); } } }
/// <summary> /// 资金流向 -大单追踪 /// </summary> public void GetLargeFundsTracking() { var url = "http://data.10jqka.com.cn/funds/ddzz/order/asc/page/{0}/ajax/1/"; ///实时更新 var httpdownloader = new HTTP(Encoding.UTF8); while (true) { if ((DateTime.Today.AddHours(9.4) < DateTime.Now && DateTime.Now < DateTime.Today.AddHours(11.4)) || (DateTime.Today.AddHours(12.9) < DateTime.Now && DateTime.Now < DateTime.Today.AddHours(15.1)) && !(DateTime.Now.DayOfWeek == DayOfWeek.Sunday || DateTime.Now.DayOfWeek == DayOfWeek.Saturday)) { for (var i = 1; i < 101; i++) { var html = httpdownloader.GetGzip(string.Format(url, i), Encoding.GetEncoding("gbk")); var data = new { ContentType = "大单追踪实时数据", Url = string.Format(url, i), CreatTime = DateTime.Now, Page = html, MD5 = Convertor.Encode_MD5(html), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageLargeFundsTracking, data); Console.WriteLine("资金流向大单追踪 已保存 " + i + " " + DateTime.Now + " " + string.Format(url, i)); } } else { Console.WriteLine("资金流向大单追踪 目前没在开市时间 工作日 9:30 - 15:00 当前时间 " + DateTime.Now); Thread.Sleep(60 * 1000); } } }
public string GetArticle(string url) { var httpDownloader = new HTTP(); var headers = new Dictionary <string, string>(); headers.Add("Accept", "*/*"); headers.Add("Accept-Encoding", "gzip, deflate, br"); headers.Add("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7"); headers.Add("Host", "www.toutiao.com"); headers.Add("User-Agent", CONST.UserAgent); var strData = httpDownloader.GetGzip(url, Encoding.UTF8, headers); if (!string.IsNullOrWhiteSpace(strData) && strData.Contains("<script>var BASE_DATA =") && strData.Contains(";</script><script>var imgUrl")) { var articleStartIndex = strData.IndexOf("<script>var BASE_DATA =") + "<script>var BASE_DATA =".Length; var articleEndIndex = strData.IndexOf(";</script><script>var imgUrl"); if (0 < articleStartIndex && articleStartIndex < articleEndIndex) { var jsonString = strData.Substring(articleStartIndex, articleEndIndex - articleStartIndex); return(jsonString); } } return(string.Empty); }
/// <summary> /// 获取搜索结果 /// </summary> /// <returns></returns> public List <string> GetSearchResult(string keyword, int maxCount = 60) { var list = new List <string>(); var count = 20; for (int offset = 0; offset < 3 * count; offset = (++offset) * count) { var url = string.Format("https://www.toutiao.com/search_content/?offset={0}&format=json&keyword={1}&autoload=true&count={2}&cur_tab=1&from=search_tab", offset, HttpUtility.UrlEncode(keyword), count); var httpDownloader = new HTTP(); var headers = new Dictionary <string, string>(); headers.Add("Accept", "application/json, text/javascript"); headers.Add("Accept-Encoding", "gzip, deflate, br"); headers.Add("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7"); headers.Add("Host", "www.toutiao.com"); headers.Add("Content-Type", "application/x-www-form-urlencoded"); headers.Add("Referer", string.Format("https://www.toutiao.com/search/?keyword={0}", HttpUtility.UrlEncode(keyword))); headers.Add("User-Agent", CONST.UserAgent); headers.Add("X-Requested-With", "XMLHttpRequest"); var strData = httpDownloader.GetGzip(url, Encoding.UTF8, headers); //strData = Convertor.FromUnicodeToUTF8(strData); list.Add(strData); LOGGER.Log(string.Format("正在获取 头条搜索结果 {0}", offset)); //ThreadManager.Pause(seconds: 5); } return(list); }
/// <summary> /// 下载个股龙虎榜明细 /// </summary> /// <param name="stockcode"></param> /// <param name="date"></param> /// <param name="rid"></param> /// <returns></returns> public string GetGGLHBMX(string stockcode, string url) { var httpdownloader = new HTTP(); //var url = string.Format("http://data.10jqka.com.cn/ifmarket/getnewlh/code/{0}/date/{1}/rid/{2}/", stockcode, date, rid); ///个股龙虎榜明细 var headers = new Dictionary <string, string>(); headers.Add("Accept", "*/*"); headers.Add("Accept-Encoding", "gzip, deflate"); headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"); headers.Add("Host", "data.10jqka.com.cn"); headers.Add("Referer", string.Format("http://data.10jqka.com.cn/market/lhbgg/code/{0}/", stockcode)); headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); headers.Add("X-Requested-With", "XMLHttpRequest"); ///下载明细 var html = httpdownloader.GetGzip(url, Encoding.GetEncoding("GBK"), headers); return(html); }
/// <summary> /// 资金流向 - 个股资金 /// </summary> /// <remarks> /// 个股资金 即时 http://data.10jqka.com.cn/funds/ggzjl/field/zdf/order/desc/page/{分页}/ajax/1/ /// 个股资金 3日 http://data.10jqka.com.cn/funds/ggzjl/board/3/field/zdf/order/desc/page/{分页}/ajax/1/ /// 个股资金 5日 http://data.10jqka.com.cn/funds/ggzjl/board/5/field/zdf/order/desc/page/{分页}/ajax/1/ /// 个股资金 10日 http://data.10jqka.com.cn/funds/ggzjl/board/10/field/zdf/order/desc/page/{分页}/ajax/1/ /// 个股资金 20日 http://data.10jqka.com.cn/funds/ggzjl/board/20/field/zdf/order/desc/page/{分页}/ajax/1/ /// /// 概念资金 即时 http://data.10jqka.com.cn/funds/gnzjl/field/new/order/asc/page/{分页}/ajax/1/ /// 概念资金 3日 http://data.10jqka.com.cn/funds/gnzjl/board/3/field/tradezdf/order/desc/page/{分页}/ajax/1/ /// 概念资金 5日 http://data.10jqka.com.cn/funds/gnzjl/board/5/field/tradezdf/order/desc/page/{分页}/ajax/1/ /// 概念资金 10日 http://data.10jqka.com.cn/funds/gnzjl/board/10/field/tradezdf/order/desc/page/{分页}/ajax/1/ /// 概念资金 20日 http://data.10jqka.com.cn/funds/gnzjl/board/20/field/tradezdf/order/desc/page/{分页}/ajax/1/ /// /// 行业资金 即时 http://data.10jqka.com.cn/funds/hyzjl/field/tradezdf/order/desc/page/{分页}/ajax/1/ /// 行业资金 3日 http://data.10jqka.com.cn/funds/hyzjl/board/3/field/tradezdf/order/desc/page/{分页}/ajax/1/ /// 行业资金 5日 http://data.10jqka.com.cn/funds/hyzjl/board/5/field/tradezdf/order/desc/page/{分页}/ajax/1/ /// 行业资金 10日 http://data.10jqka.com.cn/funds/hyzjl/board/10/field/tradezdf/order/desc/page/{分页}/ajax/1/ /// 行业资金 20日 http://data.10jqka.com.cn/funds/hyzjl/board/20/field/tradezdf/order/desc/page/{分页}/ajax/1/ /// /// 大单追踪 即时 http://data.10jqka.com.cn/funds/ddzz/order/asc/page/{分页}/ajax/1/ /// </remarks> public void GetFundsStock() { var pageCount = 64; var queueUrl = new Queue <KeyValuePair <string, string> >(); ///url,重试次数 for (int i = 1; i < pageCount; i++) { queueUrl.Enqueue(new KeyValuePair <string, string>("个股资金即时", string.Format("http://data.10jqka.com.cn/funds/ggzjl/field/zdf/order/desc/page/{0}/ajax/1/", i))); ///个股资金 即时 queueUrl.Enqueue(new KeyValuePair <string, string>("个股资金3日", string.Format("http://data.10jqka.com.cn/funds/ggzjl/board/3/field/zdf/order/desc/page/{0}/ajax/1/", i))); ///个股资金 3日 queueUrl.Enqueue(new KeyValuePair <string, string>("个股资金5日", string.Format("http://data.10jqka.com.cn/funds/ggzjl/board/5/field/zdf/order/desc/page/{0}/ajax/1/", i))); ///个股资金 5日 queueUrl.Enqueue(new KeyValuePair <string, string>("个股资金10日", string.Format("http://data.10jqka.com.cn/funds/ggzjl/board/10/field/zdf/order/desc/page/{0}/ajax/1/", i))); ///个股资金 10日 queueUrl.Enqueue(new KeyValuePair <string, string>("个股资金20日", string.Format("http://data.10jqka.com.cn/funds/ggzjl/board/20/field/zdf/order/desc/page/{0}/ajax/1/", i))); ///个股资金 20日 } for (int i = 1; i < 5; i++) { queueUrl.Enqueue(new KeyValuePair <string, string>("概念资金即时", string.Format("http://data.10jqka.com.cn/funds/gnzjl/field/new/order/asc/page/{0}/ajax/1/", i))); ///概念资金 即时 queueUrl.Enqueue(new KeyValuePair <string, string>("概念资金3日", string.Format("http://data.10jqka.com.cn/funds/gnzjl/board/3/field/tradezdf/order/desc/page/{0}/ajax/1/", i))); ///概念资金 3日 queueUrl.Enqueue(new KeyValuePair <string, string>("概念资金5日", string.Format("http://data.10jqka.com.cn/funds/gnzjl/board/5/field/tradezdf/order/desc/page/{0}/ajax/1/", i))); ///概念资金 5日 queueUrl.Enqueue(new KeyValuePair <string, string>("概念资金10日", string.Format("http://data.10jqka.com.cn/funds/gnzjl/board/10/field/tradezdf/order/desc/page/{0}/ajax/1/", i))); ///概念资金 10日 queueUrl.Enqueue(new KeyValuePair <string, string>("概念资金20日", string.Format("http://data.10jqka.com.cn/funds/gnzjl/board/20/field/tradezdf/order/desc/page/{0}/ajax/1/", i))); ///概念资金 20日 } for (int i = 1; i < 3; i++) { queueUrl.Enqueue(new KeyValuePair <string, string>("行业资金即时", string.Format("http://data.10jqka.com.cn/funds/hyzjl/field/tradezdf/order/desc/page/{0}/ajax/1/", i))); ///行业资金 即时 queueUrl.Enqueue(new KeyValuePair <string, string>("行业资金3日", string.Format("http://data.10jqka.com.cn/funds/hyzjl/board/3/field/tradezdf/order/desc/page/{0}/ajax/1/", i))); ///行业资金 3日 queueUrl.Enqueue(new KeyValuePair <string, string>("行业资金5日", string.Format("http://data.10jqka.com.cn/funds/hyzjl/board/5/field/tradezdf/order/desc/page/{0}/ajax/1/", i))); ///行业资金 5日 queueUrl.Enqueue(new KeyValuePair <string, string>("行业资金10日", string.Format("http://data.10jqka.com.cn/funds/hyzjl/board/10/field/tradezdf/order/desc/page/{0}/ajax/1/", i))); ///行业资金 10日 queueUrl.Enqueue(new KeyValuePair <string, string>("行业资金20日", string.Format("http://data.10jqka.com.cn/funds/hyzjl/board/20/field/tradezdf/order/desc/page/{0}/ajax/1/", i))); ///行业资金 20日 } var httpdownloader = new HTTP(Encoding.UTF8); httpdownloader.EventException += (object sender, EventArgs e) => { var ee = e as EventProcEventArgs; Console.WriteLine("下载失败 {0}", ee.Default); var item = ee.Default as Dictionary <string, object>; item["ContentType"] = "资金流向个股资金下载失败异常信息"; item["TaskID"] = THS.CONST.TaskID; mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item); var url = item["Url"].ToString(); var type = string.Empty; #region 判断类型 if (url.Contains("ggzjl/field")) { type = "个股资金即时"; } else if (url.Contains("/ggzjl/board/3/field")) { type = "个股资金3日"; } else if (url.Contains("ggzjl/board/5/field")) { type = "个股资金5日"; } else if (url.Contains("ggzjl/board/10/field")) { type = "个股资金10日"; } else if (url.Contains("ggzjl/board/20/field")) { type = "个股资金20日"; } else if (url.Contains("gnzjl/field/new")) { type = "概念资金即时"; } else if (url.Contains("gnzjl/board/3")) { type = "概念资金3日"; } else if (url.Contains("gnzjl/board/5")) { type = "概念资金5日"; } else if (url.Contains("gnzjl/board/10")) { type = "概念资金10日"; } else if (url.Contains("gnzjl/board/20")) { type = "概念资金20日"; } else if (url.Contains("hyzjl/field")) { type = "行业资金即时"; } else if (url.Contains("hyzjl/board/3")) { type = "行业资金3日"; } else if (url.Contains("hyzjl/board/5")) { type = "行业资金5日"; } else if (url.Contains("hyzjl/board/10")) { type = "行业资金10日"; } else if (url.Contains("hyzjl/board/20")) { type = "行业资金20日"; } #endregion queueUrl.Enqueue(new KeyValuePair <string, string>(type, url)); Thread.Sleep(60 * 1000); }; while (0 < queueUrl.Count) { var queueItem = queueUrl.Dequeue(); var html = httpdownloader.GetGzip(queueItem.Value, Encoding.GetEncoding("gbk")); var item = new { ContentType = queueItem.Key, Url = queueItem.Value, CreatTime = DateTime.Now, Remark = "次日更新", Page = html, MD5 = Convertor.Encode_MD5(html), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageFundsStock, item); Console.WriteLine("已保存 " + queueItem.Key + " " + queueItem.Value); } }
/// <summary> /// 龙虎榜个股及明细 /// </summary> public void GetStockLHB() { this.GetALLStockCode(); ///准备全部股票代码 Console.WriteLine("准备下载龙虎榜个股及明细页面"); var httpdownloader = new HTTP("gbk"); #region 准备下载队列 var queueUrl = new Queue <URL>(); ///key :url value 失败次数 var queueMxUrl = new Queue <URL>(); ///龙虎榜明细队列 foreach (var stock in this.stockCodeDict) { var urlgglhb = new URL(string.Format("http://data.10jqka.com.cn/market/lhbgg/code/{0}/", stock.Key), message: "个股龙虎榜"); queueUrl.Enqueue(urlgglhb); } #endregion var excetionDict = new Dictionary <string, int>(); while (0 < queueUrl.Count) ///下载个股龙虎榜准备龙虎榜明细队列 { var qItem = queueUrl.Dequeue(); var urlgglhb = qItem; ///个股龙虎榜URL var stockCode = urlgglhb.Url.Substring(urlgglhb.Url.Length - 8).Replace("/", string.Empty); //http://data.10jqka.com.cn/market/lhbgg/code/600360/ //http://data.10jqka.com.cn/ifmarket/getnewlh/code/601228/date/2017-04-14/rid/7/ var refID = Guid.NewGuid().ToString().Replace("-", string.Empty); var headers = new Dictionary <string, string>(); headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); headers.Add("Accept-Encoding", "gzip, deflate"); headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"); headers.Add("Host", "data.10jqka.com.cn"); headers.Add("Referer", "http://data.10jqka.com.cn/market/longhu/"); headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); var pagegglhbHtml = httpdownloader.GetGzip(urlgglhb.Url, Encoding.GetEncoding("GBK"), headers); ///下载的个股龙虎榜页面 if (100 < pagegglhbHtml.Length) ///若下载的有数据 { #region 分析个股龙虎榜明细 var tableStartIndex = pagegglhbHtml.IndexOf("<tbody>"); ///个股龙虎榜表格数据开始位置 var tableEndIndex = pagegglhbHtml.IndexOf("</tbody>"); ///个股龙虎榜表格数据结束位置 var tableHtml = pagegglhbHtml.Substring(tableStartIndex, tableEndIndex - tableStartIndex); var tdArray = tableHtml.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries); foreach (var td in tdArray) { if (td.Contains("rid") && td.Contains("date")) { var paramArray = td.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries); var date = string.Empty; var rid = string.Empty; var code = string.Empty; foreach (var param in paramArray) { if (param.Contains("code=")) { code = param.Substring(6, 6); } else if (param.Contains("date=")) { date = param.Substring(6, 10); } else if (param.Contains("rid=")) { rid = param.Substring(5, 2).Replace("\"", string.Empty); } if (!string.IsNullOrWhiteSpace(code) && !string.IsNullOrWhiteSpace(date) && !string.IsNullOrWhiteSpace(rid)) { var urlgglhbmx = string.Format("http://data.10jqka.com.cn/ifmarket/getnewlh/code/{0}/date/{1}/rid/{2}/", code, date, rid); ///个股龙虎榜明细 var mxUrl = new URL(urlgglhbmx); mxUrl.Data["StockCode"] = code; mxUrl.Data["Date"] = date; mxUrl.Data["Rid"] = rid; mxUrl.Data["ParentUrl"] = urlgglhb.Url; mxUrl.Data["RefID"] = refID; mxUrl.Message = "个股龙虎榜明细"; queueMxUrl.Enqueue(mxUrl); } } } } #endregion #region 个股龙虎榜数据保存 var gglhbData = new //个股龙虎榜 数据 { StockCode = stockCode, StockName = this.stockCodeDict[stockCode], Url = urlgglhb, CreateTime = DateTime.Now, Page = pagegglhbHtml, ContentType = "个股龙虎榜", RefID = refID,///和明细保持一致 MD5 = Convertor.Encode_MD5(pagegglhbHtml), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbData); ///个股龙虎榜 Console.WriteLine("个股龙虎榜 数据保存 {0} {1} {2}", stockCode, this.stockCodeDict[stockCode], queueUrl.Count); #endregion } else { if (excetionDict.ContainsKey(qItem.Url) && 3 <= excetionDict[qItem.Url]) { ///写异常信息 Console.WriteLine("下载失败 {0}", qItem.Url); var item = new Dictionary <string, object>(); item["ContentType"] = "龙虎榜下载失败异常信息"; item["TaskID"] = THS.CONST.TaskID; item["CreateTime"] = DateTime.Now; item["Data"] = Convertor.FromObjectToDictionary(qItem); mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item); Thread.Sleep(60 * 1000); } else { excetionDict[qItem.Url] = (excetionDict.ContainsKey(qItem.Url)) ? excetionDict[qItem.Url] + 1 : 0; queueUrl.Enqueue(qItem); } } } var excetionMxDict = new Dictionary <string, int>(); while (0 < queueMxUrl.Count) { var qItem = queueMxUrl.Dequeue(); var headers = new Dictionary <string, string>(); headers.Add("Accept", "*/*"); headers.Add("Accept-Encoding", "gzip, deflate"); headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"); headers.Add("Host", "data.10jqka.com.cn"); headers.Add("Referer", string.Format("http://data.10jqka.com.cn/market/lhbgg/code/{0}/", qItem.Data["StockCode"].ToString())); headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); headers.Add("X-Requested-With", "XMLHttpRequest"); ///下载明细 var subHtml = httpdownloader.GetGzip(qItem.Url, Encoding.GetEncoding("GBK"), headers); if (50 < subHtml.Length) { var stockCode = qItem.Data["StockCode"].ToString(); var date = qItem.Data["Date"].ToString(); var rid = qItem.Data["Rid"].ToString(); var parentUrl = qItem.Data["ParentUrl"].ToString(); var refID = qItem.Data["RefID"].ToString(); var gglhbmxData = new { StockCode = qItem.Data["StockCode"].ToString(), StockName = this.stockCodeDict[stockCode], Date = date, Rid = rid, ParentUrl = parentUrl, Url = qItem.Url, CreateTime = DateTime.Now, Page = subHtml, ContentType = "个股龙虎榜明细", RefID = refID, MD5 = Convertor.Encode_MD5(subHtml), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbmxData); Console.WriteLine("个股龙虎榜明细 数据保存 {0} {1} {2}", stockCode, this.stockCodeDict[stockCode], queueMxUrl.Count); ///若能正常下载则删除 if (excetionMxDict.ContainsKey(qItem.Url)) { ///正常下载完毕删除异常信息 excetionMxDict.Remove(qItem.Url); } } else { Console.WriteLine("下载异常 {0} ", qItem.Url); if (excetionMxDict.ContainsKey(qItem.Url) && 3 <= excetionMxDict[qItem.Url]) { ///写异常信息 Console.WriteLine("下载失败 {0}", qItem.Url); var item = new Dictionary <string, object>(); item["ContentType"] = "龙虎榜明细下载失败异常信息"; item["TaskID"] = THS.CONST.TaskID; item["CreateTime"] = DateTime.Now; item["Data"] = qItem; mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item); Thread.Sleep(60 * 1000); } else { ///重新加入队列 excetionMxDict[qItem.Url] = (excetionMxDict.ContainsKey(qItem.Url)) ? excetionMxDict[qItem.Url] + 1 : 0; queueMxUrl.Enqueue(qItem); } } } }
/// <summary> /// 下载每个股票的页面 UTF8 /// </summary> public void GetPageKLine() { this.GetALLStockCode(); ///准备所有股票代码 var queueUrl = new Queue <KeyValuePair <string, string> >(); var httpdownloader = new HTTP(Encoding.UTF8); httpdownloader.EventException += (object sender, EventArgs e) => { var ee = e as EventProcEventArgs; Console.WriteLine("下载失败 {0}", ee.Default); var item = ee.Default as Dictionary <string, object>; item["ContentType"] = "个股日线页面下载失败异常信息"; item["TaskID"] = THS.CONST.TaskID; var url = item["Url"].ToString(); var type = string.Empty; mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item); #region 类型判断 if (url.Contains("/01/last.js")) { type = "日线数据"; } #endregion queueUrl.Enqueue(new KeyValuePair <string, string>(type, url)); Thread.Sleep(60 * 1000); }; if (null != this.stockCodeDict) { foreach (var item in this.stockCodeDict) { queueUrl.Enqueue(new KeyValuePair <string, string>(item.Key.Trim(), item.Value.Trim()));///Key:StockCode Value:StockName } } while (0 < queueUrl.Count) { var queueItem = queueUrl.Dequeue(); { var urlList = new List <KeyValuePair <string, string> >(); urlList.Add(new KeyValuePair <string, string>("日线数据", string.Format("http://d.10jqka.com.cn/v2/line/hs_{0}/01/last.js", queueItem.Key))); ///日线数据 foreach (var url in urlList) { var headers = new Dictionary <string, string>(); headers.Add("Accept", "*/*"); headers.Add("Accept-Encoding", "gzip,deflate"); headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.4"); headers.Add("Host", "d.10jqka.com.cn"); headers.Add("Referer", "http://data.10jqka.com.cn/funds/ggzjl/"); headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); var html = httpdownloader.GetGzip(url.Value, Encoding.GetEncoding("GBK"), headers); var item = new { StockCode = queueItem.Key, StockName = queueItem.Value, ContentType = url.Key, Url = url.Value, CreatTime = DateTime.Now, Page = html, MD5 = Convertor.Encode_MD5(html), TaskID = THS.CONST.TaskID }; var collectionName = THS.CONST.PageKLine; mongo.Save(THS.CONST.DBName, collectionName, item); Console.WriteLine("已保存 " + url.Key + " " + url.Value); } } } }
/// <summary> /// 下载每个股票的页面 UTF8 /// </summary> public void GetPageStock() { this.GetALLStockCode(); ///准备所有股票代码 var queueUrl = new Queue <KeyValuePair <string, string> >(); var httpdownloader = new HTTP(Encoding.UTF8); httpdownloader.EventException += (object sender, EventArgs e) => { var ee = e as EventProcEventArgs; Console.WriteLine("下载失败 {0}", ee.Default); var item = ee.Default as Dictionary <string, object>; item["ContentType"] = "个股详细页面下载失败异常信息"; item["TaskID"] = THS.CONST.TaskID; var url = item["Url"].ToString(); var type = string.Empty; mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item); #region 类型判断 if (url.Contains("/funds/")) { type = "资金流向"; } else if (url.Length == "http://stockpage.10jqka.com.cn/000000/".Length) { type = "首页概览"; } #endregion queueUrl.Enqueue(new KeyValuePair <string, string>(type, url)); Thread.Sleep(60 * 1000); }; if (null != this.stockCodeDict) { foreach (var item in this.stockCodeDict) { queueUrl.Enqueue(new KeyValuePair <string, string>(item.Key.Trim(), item.Value.Trim()));///Key:StockCode Value:StockName } } while (0 < queueUrl.Count) { var queueItem = queueUrl.Dequeue(); { var urlList = new List <KeyValuePair <string, string> >(); urlList.Add(new KeyValuePair <string, string>("首页概览", string.Format("http://stockpage.10jqka.com.cn/{0}/", queueItem.Key))); ///首页概览 urlList.Add(new KeyValuePair <string, string>("资金流向", string.Format("http://stockpage.10jqka.com.cn/{0}/funds/", queueItem.Key))); ///资金流向 //urlList.Add(new KeyValuePair<string, string>("公司资料", string.Format("http://stockpage.10jqka.com.cn/{0}/company/", queueItem.Key))); ///公司资料 //urlList.Add(new KeyValuePair<string, string>("新闻公告", string.Format("http://stockpage.10jqka.com.cn/ajax/code/{0}/type/news/", queueItem.Key))); ///新闻公告 //urlList.Add(new KeyValuePair<string, string>("财务分析", string.Format("http://stockpage.10jqka.com.cn/{0}/finance/", queueItem.Key))); ///财务分析 //urlList.Add(new KeyValuePair<string, string>("经营分析", string.Format("http://stockpage.10jqka.com.cn/{0}/operate/", queueItem.Key))); ///经营分析 //urlList.Add(new KeyValuePair<string, string>("股东股本", string.Format("http://stockpage.10jqka.com.cn/{0}/holder/", queueItem.Key))); ///股东股本 //urlList.Add(new KeyValuePair<string, string>("主力持仓", string.Format("http://stockpage.10jqka.com.cn/{0}/position/", queueItem.Key))); ///主力持仓 //urlList.Add(new KeyValuePair<string, string>("公司大事", string.Format("http://stockpage.10jqka.com.cn/{0}/event/", queueItem.Key))); ///公司大事 //urlList.Add(new KeyValuePair<string, string>("分红融资", string.Format("http://stockpage.10jqka.com.cn/{0}/bonus/", queueItem.Key))); ///分红融资 //urlList.Add(new KeyValuePair<string, string>("价值分析", string.Format("http://stockpage.10jqka.com.cn/{0}/worth/", queueItem.Key))); ///价值分析 //urlList.Add(new KeyValuePair<string, string>("行业分析", string.Format("http://stockpage.10jqka.com.cn/{0}/field/", queueItem.Key))); ///行业分析 var headers = new Dictionary <string, string>(); headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); headers.Add("Accept-Encoding", "gzip,deflate"); headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"); headers.Add("Host", "stockpage.10jqka.com.cn"); headers.Add("Referer", "http://www.10jqka.com.cn/"); headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); foreach (var url in urlList) { if ("新闻公告" == url.Key) { var html = httpdownloader.GetGzip(url.Value, Encoding.GetEncoding("gbk"), headers); var item = new { StockCode = queueItem.Key, StockName = queueItem.Value, ContentType = url.Key, CreatTime = DateTime.Now, Url = url.Value, Page = html, MD5 = Convertor.Encode_MD5(html), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageStock, item); Console.WriteLine("已保存 " + url.Key + " " + url.Value); } else { var html = httpdownloader.GetGzip(url.Value, Encoding.UTF8, headers); var item = new { StockCode = queueItem.Key, StockName = queueItem.Value, ContentType = url.Key, Url = url.Value, CreatTime = DateTime.Now, Page = html, MD5 = Convertor.Encode_MD5(html), TaskID = THS.CONST.TaskID }; var collectionName = THS.CONST.PageStock; mongo.Save(THS.CONST.DBName, collectionName, item); Console.WriteLine("已保存 " + url.Key + " " + url.Value); } } } } }
/// <summary> /// 获取所有的股票代码 必须休市后再用 因为 股价变动造成排名变动 /// </summary> public void GetALLStockCode() { ///先检查本地或数据库是否有,若有就用,否则从网络获取 Console.WriteLine("初始化股票代码\t{0}", DateTime.Now); if (0 == this.stockCodeDict.Count) { var list = mongo.Find(THS.CONST.DBName, THS.CONST.StockBaseInfo, "{}"); if (0 < list.Count) ///若找得到数据 { this.stockCodeDict.Clear(); foreach (var item in list) { this.stockCodeDict.Add(item["StockCode"].ToString(), item["StockName"].ToString()); } Console.WriteLine("股票基本信息从数据库初始化完毕"); } else { Console.WriteLine("重新获取股票信息"); var headers = new Dictionary <string, string>(); headers.Add("Accept", "text/html,*/*; q=0.01"); headers.Add("Accept-Encoding", "gzip, deflate"); headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"); headers.Add("Host", "q.10jqka.com.cn"); headers.Add("Referer", "http://q.10jqka.com.cn/"); headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); headers.Add("X-Requested-With", "XMLHttpRequest"); #region 获取分页数 var pageCount = 0; var pagerUrl = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", 1); var pagerHtml = http.GetGzip(pagerUrl, Encoding.GetEncoding("GBK"), headers); if (100 < pagerHtml.Length) { pageCount = int.Parse(pagerHtml.Substring(pagerHtml.LastIndexOf("</span>") - 3, 3)); ///页码数 Console.WriteLine("股票代码页数{0}\t{1}", pageCount, DateTime.Now); } #endregion var stringBuilder = new StringBuilder(); for (int i = 1; i <= pageCount; i++) { var url = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", i); var html = http.GetGzip(url, Encoding.GetEncoding("GBK"), headers); #region 股票代码处理 string[] trArray = html.Substring(html.IndexOf("<tbody>") + "<tbody>".Length).Split(new string[] { "<tr>", "</tr>" }, StringSplitOptions.RemoveEmptyEntries); foreach (var tr in trArray) { var tdArray = tr.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries); if (10 < tdArray.Length) { var stockCode = tdArray[3].Substring(tdArray[3].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", ""); var stockName = tdArray[5].Substring(tdArray[5].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", ""); if (!stockCodeDict.ContainsKey(stockCode) && !stockCodeDict.ContainsValue(stockName)) { stockCodeDict.Add(stockCode, stockName); } else { throw new Exception("数据重复"); } Console.WriteLine("正在添加 {0}\t{1}", stockCode, stockName); } } #endregion } #region 输出检查 foreach (var item in stockCodeDict) { Console.WriteLine(item.Key + " " + item.Value.Replace(" ", string.Empty)); stringBuilder.AppendLine(item.Key + "\t" + item.Value.Replace(" ", string.Empty)); } #endregion #region 文件保存 File.WriteAllText(@"E:\stockCode.txt", stringBuilder.ToString(), Encoding.UTF8); Console.WriteLine("stockCode.txt 保存完毕"); #endregion #region 保存到数据库 foreach (var item in stockCodeDict) { mongo.Save(THS.CONST.DBName, THS.CONST.StockBaseInfo, new { StockCode = item.Key, StockName = item.Value, CreateTime = DateTime.Now, UpdateTime = DateTime.Now, TaskID = THS.CONST.TaskID }); } #endregion Console.WriteLine("股票代码添加完毕 {0}", DateTime.Now); } } }
/// <summary> /// 龙虎榜个股及明细 /// </summary> public void GetStockLHB() { this.GetALLStockCode(); ///准备全部股票代码 Console.WriteLine("准备下载龙虎榜个股及明细页面"); var httpdownloader = new HTTP("gbk"); #region 准备下载队列 var queueUrl = new Queue <string>(); ///key :url value 失败次数 foreach (var stock in this.stockCodeDict) { var urlgglhb = string.Format("http://data.10jqka.com.cn/market/lhbgg/code/{0}/", stock.Key);// queueUrl.Enqueue(urlgglhb); } #endregion ///若下载失败 httpdownloader.EventException += (object sender, EventArgs e) => { var ee = e as EventProcEventArgs; Console.WriteLine("下载失败 {0}", ee.Default); var item = ee.Default as Dictionary <string, object>; item["ContentType"] = "龙虎榜下载失败异常信息"; item["TaskID"] = THS.CONST.TaskID; mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item); queueUrl.Enqueue(item["Url"].ToString()); Thread.Sleep(60 * 1000); }; Func <string, string, string, string, string, int> downloadLHBMX = (string code, string date, string rid, string urlgglhb, string refID) => { var urlgglhbmx = string.Format("http://data.10jqka.com.cn/ifmarket/getnewlh/code/{0}/date/{1}/rid/{2}/", code, date, rid); ///个股龙虎榜明细 var subHtml = httpdownloader.GetGzip(urlgglhbmx); var gglhbmxData = new { StockCode = code, StockName = this.stockCodeDict[code], Date = date, Rid = rid, ParentUrl = urlgglhb, Url = urlgglhbmx, CreateTime = DateTime.Now, Page = subHtml, ContentType = "个股龙虎榜明细", RefID = refID, MD5 = Convertor.Encode_MD5(subHtml), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbmxData); return(0); }; while (0 < queueUrl.Count) { var qItem = queueUrl.Dequeue(); var urlgglhb = qItem; ///个股龙虎榜URL var url = qItem; var stockCode = urlgglhb.Substring(urlgglhb.Length - 8).Replace("/", string.Empty); //http://data.10jqka.com.cn/market/lhbgg/code/600360/ //http://data.10jqka.com.cn/ifmarket/getnewlh/code/601228/date/2017-04-14/rid/7/ var refID = Guid.NewGuid().ToString().Replace("-", string.Empty); var pagegglhbHtml = httpdownloader.GetGzip(urlgglhb); ///下载的个股龙虎榜页面 if (100 < pagegglhbHtml.Length) ///若下载的有数据 { #region 分析个股龙虎榜明细 var tableStartIndex = pagegglhbHtml.IndexOf("<tbody>"); ///个股龙虎榜表格数据开始位置 var tableEndIndex = pagegglhbHtml.IndexOf("</tbody>"); ///个股龙虎榜表格数据结束位置 var tableHtml = pagegglhbHtml.Substring(tableStartIndex, tableEndIndex - tableStartIndex); var tdArray = tableHtml.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries); foreach (var td in tdArray) { if (td.Contains("rid") && td.Contains("date")) { var paramArray = td.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries); var date = string.Empty; var rid = string.Empty; var code = string.Empty; foreach (var param in paramArray) { if (param.Contains("code=")) { code = param.Substring(6, 6); } else if (param.Contains("date=")) { date = param.Substring(6, 10); } else if (param.Contains("rid=")) { rid = param.Substring(5, 2).Replace("\"", string.Empty); } if (!string.IsNullOrWhiteSpace(code) && !string.IsNullOrWhiteSpace(date) && !string.IsNullOrWhiteSpace(rid)) { var urlgglhbmx = string.Format("http://data.10jqka.com.cn/ifmarket/getnewlh/code/{0}/date/{1}/rid/{2}/", code, date, rid); ///个股龙虎榜明细 var subHtml = httpdownloader.GetGzip(urlgglhbmx); var gglhbmxData = new { StockCode = code, StockName = this.stockCodeDict[code], Date = date, Rid = rid, ParentUrl = urlgglhb, Url = urlgglhbmx, CreateTime = DateTime.Now, Page = subHtml, ContentType = "个股龙虎榜明细", RefID = refID, MD5 = Convertor.Encode_MD5(subHtml), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbmxData); } } } } #endregion #region 数据保存 var gglhbData = new //个股龙虎榜 数据 { StockCode = stockCode, StockName = this.stockCodeDict[stockCode], Url = urlgglhb, CreateTime = DateTime.Now, Page = pagegglhbHtml, ContentType = "个股龙虎榜", RefID = refID, MD5 = Convertor.Encode_MD5(pagegglhbHtml), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageGGLHB, gglhbData); ///个股龙虎榜 Console.WriteLine("个股龙虎榜 数据保存 {0} {1} {2}", stockCode, this.stockCodeDict[stockCode], queueUrl.Count); #endregion } else { var item = new { ContentType = "龙虎榜下载数据异常", Page = pagegglhbHtml, Url = urlgglhb, CreateTime = DateTime.Now, MD5 = Convertor.Encode_MD5(pagegglhbHtml), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item); } } }
/// <summary> /// 下载每个股票的页面 UTF8 /// </summary> public void GetPageStock() { this.GetALLStockCode(); ///准备所有股票代码 var queueUrl = new Queue <KeyValuePair <string, string> >(); var httpdownloader = new HTTP(Encoding.UTF8); httpdownloader.EventException += (object sender, EventArgs e) => { var ee = e as EventProcEventArgs; Console.WriteLine("下载失败 {0}", ee.Default); var item = ee.Default as Dictionary <string, object>; item["ContentType"] = "个股详细页面下载失败异常信息"; item["TaskID"] = THS.CONST.TaskID; var url = item["Url"].ToString(); var type = string.Empty; mongo.Save(THS.CONST.DBName, THS.CONST.Exception, item); #region 类型判断 if (url.Contains("/funds/")) { type = "资金流向"; } if (url.Contains("/01/last.js")) { type = "日线数据"; } else if (url.Length == "http://stockpage.10jqka.com.cn/000000/".Length) { type = "首页概览"; } #endregion queueUrl.Enqueue(new KeyValuePair <string, string>(type, url)); Thread.Sleep(60 * 1000); }; if (null != this.stockCodeDict) { foreach (var item in this.stockCodeDict) { queueUrl.Enqueue(new KeyValuePair <string, string>(item.Key.Trim(), item.Value.Trim()));///Key:StockCode Value:StockName } } while (0 < queueUrl.Count) { var queueItem = queueUrl.Dequeue(); { var urlList = new List <KeyValuePair <string, string> >(); urlList.Add(new KeyValuePair <string, string>("首页概览", string.Format("http://stockpage.10jqka.com.cn/{0}/", queueItem.Key))); ///首页概览 urlList.Add(new KeyValuePair <string, string>("资金流向", string.Format("http://stockpage.10jqka.com.cn/{0}/funds/", queueItem.Key))); ///资金流向 //urlList.Add(new KeyValuePair<string, string>("公司资料", string.Format("http://stockpage.10jqka.com.cn/{0}/company/", queueItem.Key))); ///公司资料 //urlList.Add(new KeyValuePair<string, string>("新闻公告", string.Format("http://stockpage.10jqka.com.cn/ajax/code/{0}/type/news/", queueItem.Key))); ///新闻公告 //urlList.Add(new KeyValuePair<string, string>("财务分析", string.Format("http://stockpage.10jqka.com.cn/{0}/finance/", queueItem.Key))); ///财务分析 //urlList.Add(new KeyValuePair<string, string>("经营分析", string.Format("http://stockpage.10jqka.com.cn/{0}/operate/", queueItem.Key))); ///经营分析 //urlList.Add(new KeyValuePair<string, string>("股东股本", string.Format("http://stockpage.10jqka.com.cn/{0}/holder/", queueItem.Key))); ///股东股本 //urlList.Add(new KeyValuePair<string, string>("主力持仓", string.Format("http://stockpage.10jqka.com.cn/{0}/position/", queueItem.Key))); ///主力持仓 //urlList.Add(new KeyValuePair<string, string>("公司大事", string.Format("http://stockpage.10jqka.com.cn/{0}/event/", queueItem.Key))); ///公司大事 //urlList.Add(new KeyValuePair<string, string>("分红融资", string.Format("http://stockpage.10jqka.com.cn/{0}/bonus/", queueItem.Key))); ///分红融资 //urlList.Add(new KeyValuePair<string, string>("价值分析", string.Format("http://stockpage.10jqka.com.cn/{0}/worth/", queueItem.Key))); ///价值分析 //urlList.Add(new KeyValuePair<string, string>("行业分析", string.Format("http://stockpage.10jqka.com.cn/{0}/field/", queueItem.Key))); ///行业分析 urlList.Add(new KeyValuePair <string, string>("日线数据", string.Format("http://d.10jqka.com.cn/v2/line/hs_{0}/01/last.js", queueItem.Key))); ///日线数据 //urlList.Add(new KeyValuePair<string, string>("日线数据", string.Format("http://d.10jqka.com.cn/v6/line/hs_{0}/01/all.js", queueItem.Key))); ///日线数据 foreach (var url in urlList) { if ("新闻公告" == url.Key) { var html = httpdownloader.GetGzip(url.Value, Encoding.GetEncoding("gbk")); var item = new { StockCode = queueItem.Key, StockName = queueItem.Value, ContentType = url.Key, CreatTime = DateTime.Now, Url = url.Value, Page = html, MD5 = Convertor.Encode_MD5(html), TaskID = THS.CONST.TaskID }; mongo.Save(THS.CONST.DBName, THS.CONST.PageStock, item); Console.WriteLine("已保存 " + url.Key + " " + url.Value); } else { var html = httpdownloader.GetGzip(url.Value, Encoding.UTF8); var item = new { StockCode = queueItem.Key, StockName = queueItem.Value, ContentType = url.Key, Url = url.Value, CreatTime = DateTime.Now, Page = html, MD5 = Convertor.Encode_MD5(html), TaskID = THS.CONST.TaskID }; var collectionName = ("日线数据" == url.Key) ? THS.CONST.PageKLine : THS.CONST.PageStock; mongo.Save(THS.CONST.DBName, collectionName, item); Console.WriteLine("已保存 " + url.Key + " " + url.Value); } } } } }
/// <summary> /// 获取所有的股票代码 必须休市后再用 因为 股价变动造成排名变动 /// </summary> public void GetALLStockCode() { ///先检查本地或数据库是否有,若有就用,否则从网络获取 Console.WriteLine("初始化股票代码\t{0}", DateTime.Now); if (0 == this.stockCodeDict.Count) { var list = mongo.Find(THS.CONST.DBName, THS.CONST.StockBaseInfo, "{}"); if (0 < list.Count) ///若找得到数据 { this.stockCodeDict.Clear(); foreach (var item in list) { this.stockCodeDict.Add(item["StockCode"].ToString(), item["StockName"].ToString()); } Console.WriteLine("股票基本信息从数据库初始化完毕"); } else { Console.WriteLine("重新获取股票信息"); #region 获取分页数 var pageCount = 0; var pagerUrl = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", 1); var pagerHtml = http.GetGzip(pagerUrl); if (100 < pagerHtml.Length) { pageCount = int.Parse(pagerHtml.Substring(pagerHtml.LastIndexOf("</span>") - 3, 3)); ///页码数 Console.WriteLine("股票代码页数{0}\t{1}", pageCount, DateTime.Now); } #endregion var stringBuilder = new StringBuilder(); for (int i = 1; i <= pageCount; i++) { var url = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", i); var html = http.GetGzip(url); #region 股票代码处理 string[] trArray = html.Substring(html.IndexOf("<tbody>") + "<tbody>".Length).Split(new string[] { "<tr>", "</tr>" }, StringSplitOptions.RemoveEmptyEntries); foreach (var tr in trArray) { var tdArray = tr.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries); if (10 < tdArray.Length) { var stockCode = tdArray[3].Substring(tdArray[3].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", ""); var stockName = tdArray[5].Substring(tdArray[5].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", ""); if (!stockCodeDict.ContainsKey(stockCode) && !stockCodeDict.ContainsValue(stockName)) { stockCodeDict.Add(stockCode, stockName); } else { throw new Exception("数据重复"); } Console.WriteLine("正在添加 {0}\t{1}", stockCode, stockName); } } #endregion } #region 输出检查 foreach (var item in stockCodeDict) { Console.WriteLine(item.Key + " " + item.Value.Replace(" ", string.Empty)); stringBuilder.AppendLine(item.Key + "\t" + item.Value.Replace(" ", string.Empty)); } #endregion #region 文件保存 File.WriteAllText(@"E:\stockCode.txt", stringBuilder.ToString(), Encoding.UTF8); Console.WriteLine("stockCode.txt 保存完毕"); #endregion #region 保存到数据库 foreach (var item in stockCodeDict) { mongo.Save(THS.CONST.DBName, THS.CONST.StockBaseInfo, new { StockCode = item.Key, StockName = item.Value, CreateTime = DateTime.Now, UpdateTime = DateTime.Now, TaskID = THS.CONST.TaskID }); } #endregion Console.WriteLine("股票代码添加完毕 {0}", DateTime.Now); } } }