/// <summary> /// SINA融资融券 /// </summary> public void SyncRZRQ() { var startTime = DateTime.Now;///开始运行时间 Console.Title = "SINA融资融券 更新进程 启动时间:" + startTime; var exe = StockTaskExecutor.CreateInstance(); var mongo = DataStorage.GetInstance(DBType.MongoDB); var dbName = CONST.DB.DBName_StockService; var collectionNameRZRQ = CONST.DB.CollectionName_RZRQ; var methodName = "SyncRZRQ"; while (true) { var q = this.PrepareData(methodName); while (0 < q.Count) { var stockCode = q.Dequeue(); var stockName = this.stockCodeDict[stockCode]; var array = WebDataSource.GetInstance().GetRZRQ(stockCode); foreach (Dictionary <string, object> arrayItem in array) { var svItem = new { StockCode = stockCode, StockName = stockName, TradingDate = DateTime.Parse(arrayItem["日期"].ToString()), RZYE = arrayItem["融资余额"], RZMRE = arrayItem["融资买入额"], RZCHE = arrayItem["融资偿还额"], RQYLJE = arrayItem["融券余量金额"], RQYL = arrayItem["融券余量"], RQMCL = arrayItem["融券卖出量"], RQCHL = arrayItem["融券偿还量"], RQYE = arrayItem["融券余额"], CreateTime = DateTime.Now }; LOGGER.Log(string.Format("SINA融资融券 {0} {1} {2} 保存完毕", stockCode, stockName, svItem.TradingDate)); var query = "{\"StockCode\":\"" + stockCode + "\",\"TradingDate\":new Date('" + string.Format("{0:yyyy/MM/dd}", svItem.TradingDate) + "')}"; mongo.Save3(dbName, collectionNameRZRQ, svItem, query); } ThreadManager.Pause(seconds: 2); TaskStatusManager.Set(methodName, new { ID = methodName, StockCode = stockCode, StockName = stockName, Status = "已下载", CreateTime = DateTime.Now }); LOGGER.Log(string.Format("SINA融资融券 {0} {1} 保存完毕", stockCode, stockName)); } LOGGER.Log(string.Format("本次SINA融资融券更新完毕,下一次一天以后更新 {0}", DateTime.Now)); TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now }); ThreadManager.Pause(days: 1); } }
/// <summary> /// SINA 公司简介 /// </summary> public void SyncGSJJ() { var startTime = DateTime.Now;///开始运行时间 Console.Title = "SINA公司简介 更新进程 启动时间:" + startTime; var exe = StockTaskExecutor.CreateInstance(); var mongo = DataStorage.GetInstance(DBType.MongoDB); var dbName = CONST.DB.DBName_StockService; var collectionName = CONST.DB.CollectionName_GSJJ; var methodName = "SyncGSJJ"; while (true) { ///获取所有股票代码,遍历更新数据 var q = this.PrepareData(methodName); while (0 < q.Count) { var stockCode = q.Dequeue(); var resObj = WebDataSource.GetInstance().GetGSJJ(stockCode); var resDict = (resObj is Dictionary <string, object>) ? (resObj as Dictionary <string, object>)["PageData"] : new Dictionary <string, object>(); ///保存到数据库中 二维化 var svItem = new { StockCode = stockCode, StockName = this.stockCodeDict[stockCode], Url = string.Format("http://vip.stock.finance.sina.com.cn/corp/go.php/vCI_CorpInfo/stockid/{0}.phtml", stockCode), CreateTime = DateTime.Now, PageData = resDict }; ///删除旧数据 ///添加新数据 LOGGER.Log(string.Format("更新{0} {1}的公司简介 ", stockCode, svItem.StockName)); var filter = "{\"ContentType\":\"SINA公司简介\",\"StockCode\":\"" + stockCode + "\"}"; mongo.Save3(dbName, collectionName, svItem, filter); TaskStatusManager.Set(methodName, new { ID = methodName, StockCode = stockCode, StockName = svItem.StockName, Status = "已下载", CreateTime = DateTime.Now }); ThreadManager.Pause(seconds: 5); } TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now }); LOGGER.Log(string.Format("本次板块概念更新完毕,下一次一天以后更新 {0}", DateTime.Now)); ThreadManager.Pause(days: 2); q = this.PrepareData(methodName); } }
public void SyncStockRardar() { var startTime = DateTime.Now;///开始运行时间 Console.Title = "SINA 股市雷达异动 更新进程 启动时间:" + startTime; var exe = StockTaskExecutor.CreateInstance(); var mongo = DataStorage.GetInstance(DBType.MongoDB); var dbName = CONST.DB.DBName_StockService; var collectionName = CONST.DB.CollectionName_Radar; var methodName = "SyncStockRardar"; while (true) { var filterCheck = "{\"TradingDate\":new Date('" + string.Format("{0:yyyy/MM/dd}", DateTime.Now) + "')}"; var checkRes = mongo.Count(dbName, collectionName, filterCheck); if (0 == checkRes && CONST.IsSafeUpdateTime(1)) { var listData = WebDataSource.GetInstance().GetStockRadar(); foreach (var item in listData) { var tradingTime = item["异动时间"].ToString().Replace("-", "/"); var filter = "{\"ContentType\":\"SINA 股市雷达\",\"TradingTime\":new Date('" + tradingTime + "')}"; var svItem = new { StockCode = item["股票代码"], StockName = item["股票简称"], AnomalyInfo = item["异动信息"], TradingTime = item["异动时间"], TradingDate = Convert.ToDateTime(tradingTime).Date }; mongo.Save3(dbName, collectionName, svItem, filter); } TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now }); LOGGER.Log(string.Format("本次股市雷达异动更新完毕,下一次一天以后更新 {0}", DateTime.Now)); } else { LOGGER.Log(string.Format("今日股市雷达异动已经更新,下一次一天以后更新")); } ThreadManager.Pause(hours: 2); } }
/// <summary> /// SINA融资融券 /// </summary> public void SyncTouTiao() { var startTime = DateTime.Now;///开始运行时间 Console.Title = "股票全网新闻 更新进程 启动时间:" + startTime; var exe = StockTaskExecutor.CreateInstance(); var mongo = DataStorage.GetInstance(DBType.MongoDB); var dbName = CONST.DB.DBName_StockService; var collectionNameRZRQ = CONST.DB.CollectionName_RZRQ; var methodName = "SyncTouTiao"; while (true) { var q = this.PrepareData(methodName); while (0 < q.Count) { var stockCode = q.Dequeue(); var stockName = this.stockCodeDict[stockCode]; var array = WebDataSource.GetInstance().GetTouTiaoSearch(stockName); foreach (Dictionary <string, object> arrayItem in array) { var doc = DocItem.Create(arrayItem); doc.Save(); LOGGER.Log(string.Format("股票全网新闻 {0} {1} 保存完毕", stockCode, stockName)); } ThreadManager.Pause(seconds: 1); TaskStatusManager.Set(methodName, new { ID = methodName, StockCode = stockCode, StockName = stockName, Status = "已下载", CreateTime = DateTime.Now }); LOGGER.Log(string.Format("股票全网新闻 {0} {1} 保存完毕", stockCode, stockName)); } LOGGER.Log(string.Format("股票全网新闻更新完毕,下一次一天以后更新 {0}", DateTime.Now)); TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now }); ThreadManager.Pause(days: 1); } }
/// <summary> /// THS龙虎榜 /// </summary> public void SyncLHB() { var startTime = DateTime.Now;///开始运行时间 Console.Title = "THS龙虎榜 更新进程 启动时间:" + startTime; var exe = StockTaskExecutor.CreateInstance(); var mongo = DataStorage.GetInstance(DBType.MongoDB); var dbName = CONST.DB.DBName_StockService; var collectionNameLHB = CONST.DB.CollectionName_LHB; var collectionNameLHBMX = CONST.DB.CollectionName_LHBMX; var methodName = "SyncLHB"; while (true) { var q = this.PrepareData(methodName); while (0 < q.Count) { var stockCode = q.Dequeue(); var stockName = this.stockCodeDict[stockCode]; var listData = WebDataSource.GetInstance().GetLHB(stockCode); var lhb = listData.First() as Dictionary <string, object>; var lhbColumn = lhb["Column"] as Dictionary <string, object>; var lhbData = lhb["Data"] as ArrayList; var tradingDate = DateTime.MinValue; foreach (Dictionary <string, object> itemLHB in lhbData) { tradingDate = DateTime.Parse(itemLHB["C1"].ToString()); var queryLHB = "{ \"StockCode\":\"" + stockCode + "\", \"TradingDate\":new Date('" + string.Format("{0:yyyy/MM/dd}", tradingDate) + "')}"; var svItem = new { StockCode = stockCode, StockName = stockName, TradingDate = tradingDate, Data = itemLHB }; mongo.Save3(dbName, collectionNameLHB, svItem, queryLHB); } listData.RemoveAt(0); foreach (Dictionary <string, object> lhbmx in listData) ///第一个是龙虎榜,后面的是龙虎榜明细 { var queryLHBMX = "{ \"StockCode\":\"" + stockCode + "\", \"TradingDate\":new Date('" + string.Format("{0:yyyy/MM/dd}", tradingDate) + "')}"; var svItem = new { StockCode = stockCode, StockName = stockName, TradingDate = tradingDate, Data = lhbmx }; mongo.Save3(dbName, collectionNameLHBMX, svItem, queryLHBMX); } TaskStatusManager.Set(methodName, new { ID = methodName, StockCode = stockCode, StockName = stockName, Status = "已下载", CreateTime = DateTime.Now }); LOGGER.Log(string.Format("本次THS龙虎榜更新完毕,下一次一天以后更新 {0}", DateTime.Now)); ThreadManager.Pause(days: 1); } TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now }); } }
/// <summary> /// THS资金流向 /// </summary> public void SyncZJLX() { var startTime = DateTime.Now;///开始运行时间 Console.Title = "THS资金流向 更新进程 启动时间:" + startTime; var exe = StockTaskExecutor.CreateInstance(); var mongo = DataStorage.GetInstance(DBType.MongoDB); var dbName = CONST.DB.DBName_StockService; var collectionName = CONST.DB.CollectionName_ZJLX; var methodName = "SyncZJLX"; while (true) { ///获取所有股票代码,遍历更新数据 var q = this.PrepareData(methodName); while (0 < q.Count) { var stockCode = q.Dequeue(); var resObj = WebDataSource.GetInstance().GetZJLX(stockCode); var resList = (resObj is Dictionary <string, object>) ? (resObj as Dictionary <string, object>)["PageData"] : new ArrayList(); var stockName = this.stockCodeDict[stockCode]; ///二维化 if (resList is ArrayList) { var rows = (resList as ArrayList); foreach (Dictionary <string, object> row in rows) { var svItem2D = new { StockCode = stockCode, StockName = stockName, Url = string.Format("http://stockpage.10jqka.com.cn/{0}/funds/", stockCode), CreateTime = DateTime.Now, TradingDate = DateTime.Parse(row["日期"].ToString()), Close = row["收盘价"], Increase = row["涨跌幅"], NetInflow = row["资金净流入"], NetLarge5Day = row["5日主力净额"], NetLarge = row["大单(主力)净额"], NetLargeProportion = row["大单(主力)净占比"], NetMedium = row["中单净额"], NetMediumProportion = row["中单净占比"], NetSmall = row["小单净额"], NetSmallProportion = row["小单净占比"], }; var filter = "{\"ContentType\":\"THS资金流向\",\"StockCode\":\"" + stockCode + "\",\"TradingDate\":new Date('" + string.Format("{0:yyyy/MM/dd}", svItem2D.TradingDate) + "')}"; mongo.Save3(dbName, collectionName, svItem2D, filter); LOGGER.Log(string.Format("THS资金流向 正在更新 {0} {1} {2}", svItem2D.StockCode, svItem2D.StockName, svItem2D.TradingDate)); } } ThreadManager.Pause(seconds: 5); TaskStatusManager.Set(methodName, new { ID = methodName, StockCode = stockCode, StockName = stockName, Status = "已下载", CreateTime = DateTime.Now }); ///同步到SQL数据库中 } TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now }); ThreadManager.Pause(days: 1); q = this.PrepareData(); } }
/// <summary> /// 同步日线数据 /// </summary> public void SyncKLineDay() { var mongo = DataStorage.GetInstance(DBType.MongoDB); var exe = WebDataSource.GetInstance();// var dbName = CONST.DB.DBName_StockService; var collectionName = CONST.DB.CollectionName_KLine; var methodName = "SyncKLineDay"; while (true) { var q = this.PrepareData(methodName); var year = DateTime.Now.Year; var jidu = Convertor.GetJidu(DateTime.Now); ///获取当前季度数据 while (0 < q.Count) { var stockCode = q.Dequeue(); var stockName = this.stockCodeDict[stockCode]; var resDict = exe.GetSINAKLineDay(stockCode, year, jidu); if (null != resDict && resDict.ContainsKey("Rows") && (resDict["Rows"] is ArrayList)) { var rows = resDict["Rows"] as ArrayList; if (rows is ArrayList) { for (var k = 0; k < rows.Count; k++) { var row = rows[k] as Dictionary <string, object>; var prevClose = 0.0f; if (1 <= k) { prevClose = Convert.ToSingle((rows[k - 1] as Dictionary <string, object>)["收盘价"]); } var svItem2D = new { StockCode = stockCode, StockName = this.stockCodeDict[stockCode], ContentType = "SINA历史交易", Year = year, CreateTime = DateTime.Now, UpdateTime = DateTime.Now, TradingDate = Convert.ToDateTime(row["日期"]), Open = row["开盘价"], High = row["最高价"], Close = row["收盘价"], Low = row["最低价"], Volume = row["交易量(股)"], Turnover = row["交易金额(元)"], Amplitude = (k == 0) ? -1 : (Convert.ToSingle(row["最高价"]) - Convert.ToSingle(row["最低价"])) / prevClose, ///振幅 Increase = (k == 0) ? -1 : ((Convert.ToSingle(row["收盘价"]) - prevClose) / prevClose), ///涨幅 AveragePrice = Convert.ToSingle(row["交易金额(元)"]) / Convert.ToSingle(row["交易量(股)"]), ///成交均价 RowIndex = k, RowCount = rows.Count }; ///状态保存 var filter = "{\"StockCode\":\"" + stockCode + "\",\"TradingDate\":new Date('" + string.Format("{0:yyyy/MM/dd}", svItem2D.TradingDate) + "')}"; mongo.Save3(dbName, collectionName, svItem2D, filter); TaskStatusManager.Set(methodName, new { ID = methodName, StockCode = stockCode, StockName = svItem2D.StockName, Status = "已下载", CreateTime = DateTime.Now }); LOGGER.Log(string.Format("保存 {0} {1} SINA历史交易2D", stockCode, stockName)); } } } ThreadManager.Pause(seconds: 5); TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now }); ///本季度下载完毕之后再下上一季度数据。 if (0 == q.Count && jidu == Convertor.GetJidu(DateTime.Now) && 1 < jidu)//当年2,3,4季度 当季度 { jidu = jidu - 1; q = this.PrepareData(methodName); } else if (0 == q.Count && jidu == Convertor.GetJidu(DateTime.Now) && 1 == jidu) { jidu = 4; ///上一年第四季度 year = year - 1; ///上一年 q = this.PrepareData(methodName); } #region 补充计算缺失 #endregion } LOGGER.Log("历史交易获取完毕 ,下一次将在24小时后开始"); ThreadManager.Pause(days: 1); } }
/// <summary> /// 获取指定日期的新闻 /// </summary> public void GetNewsListCJYW(string dateTime) { if (StringChecker.IsDateTime(dateTime)) { var formatDate = string.Format("{0:yyyyMMdd}", Convert.ToDateTime(dateTime)); var mongo = DataStorage.GetInstance(DBType.MongoDB); var webSrc = DataSourceTHS.CreateInstance(); var newNewsList = new Dictionary <string, object>(); var listHtml = webSrc.GetNewsListCJYW(Convert.ToDateTime(dateTime)); var dbName = CONST.DB.DBName_StockService; var collectionNews = CONST.DB.CollectionName_News; if (!string.IsNullOrWhiteSpace(listHtml)) { var resDict = NodeService.Get(CONST.NodeServiceUrl, "同花顺", "GetDataFromHtml", new { ContentType = "THS财经要闻新闻列表", Page = listHtml }) as Dictionary <string, object>; var resList = resDict["PageData"] as ArrayList; if (null != resList) { foreach (Dictionary <string, object> listItem in resList) { var href = listItem["Href"].ToString().Trim(); var parentUrl = string.Format("http://news.10jqka.com.cn/today_list/{0}/", formatDate).Trim(); var newsHtml = webSrc.GetNewsArticle(href, parentUrl); if (!string.IsNullOrWhiteSpace(newsHtml)) { TaskStatusManager.Set("GetNewsListCJYW", new { ID = "GetNewsListCJYW", CreateTime = DateTime.Now, Status = "准备操作THS财经要闻新闻详细", Html = newsHtml }); var resItem = NodeService.Get(CONST.NodeServiceUrl, "同花顺", "GetDataFromHtml", new { ContentType = "THS财经要闻新闻详细", Page = newsHtml }) as Dictionary <string, object>; var resDetail = resItem["PageData"] as Dictionary <string, object>; if (null != resDetail) { var newsCreateTimeString = resDetail["CreateTime"].ToString().Trim(); var svItem = new { ContentType = "THS财经要闻", Title = resDetail["Title"].ToString().Trim(), Url = href, ParentUrl = parentUrl, SourceHref = resDetail["SourceHref"].ToString().Trim(), SourceName = resDetail["SourceName"].ToString().Trim(), NewsCreateTime = (StringChecker.IsDateTime(newsCreateTimeString))?Convert.ToDateTime(newsCreateTimeString):DateTime.MinValue, Content = resDetail["Content"].ToString().Trim(), Tag = Convert.ToInt32(formatDate), CreateTime = DateTime.Now, PageMD5 = "无" }; newNewsList.Add(href, svItem); LOGGER.Log(string.Format("获取一个新闻正文 {0}", svItem.Title)); ThreadManager.Pause(seconds: 5); ///5秒钟更新一次新闻 ///分词 var task = Task.Factory.StartNew(() => { var fcZStartTime = DateTime.Now; this.SaveFenCi(svItem.Content, svItem.Url); LOGGER.Log(string.Format("分词花费时间 开始时间:{0} 花费时间:{1}", fcZStartTime, DateTime.Now - fcZStartTime)); }); } } } foreach (var svItem in newNewsList) { var filter = "{\"Url\":\"" + svItem.Key + "\"}"; mongo.Save3(dbName, collectionNews, svItem.Value, filter); } } } else { LOGGER.Log(string.Format("获取的新闻列表为空白")); } } else { LOGGER.Log(string.Format("传入的时间字符串不对:{0}", dateTime)); ThreadManager.Pause(minutes: 5); } }
/// <summary> /// 下载大单数据 /// </summary> public void GetDaDanData() { ///获取页码码数 ///下载每一页数据 ///查找最近一个交易日的数据是否存在,若存在则不下载 /// var mongo = DataStorage.GetInstance(DBType.MongoDB); var webSource = DataSourceSINA.GetInstance(); var pageCount = 0;// webSource.GetDaDanPageCount(); if (0 == pageCount) { pageCount = int.Parse(WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceSINA&m=GetDaDanPageCount"))); } var dbName = CONST.DB.DBName_StockService; var collectionName = CONST.DB.CollectionName_DaDan; ///数据检查 var lastTradingDate = Convertor.CalTradingDate(DateTime.Now, "00:00:00"); ///删除根据日期创建的临时表格 ///将数据写入表格 ///将这个数据转移到大表中 #region 存储临时数据 var tempCollectionName = collectionName + string.Format("{0:yyyyMMdd}", DateTime.Now); mongo.Remove(dbName, tempCollectionName, "{}"); #endregion var methodName = "GetDaDanData"; var status = TaskStatusManager.Get(methodName); var startIndex = (status.ContainsKey("PageIndex")) ? int.Parse(status["PageIndex"].ToString()) : 0; var totalCount = 0; for (int i = startIndex; i < pageCount; i++) { var html = "";// webSource.GetDaDan(i); if (string.IsNullOrWhiteSpace(html)) { html = WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceSINA&m=GetDaDan&p={0}", i)); } var res = NodeService.Get(CONST.NodeServiceUrl, "新浪", "GetDataFromHtml", new { ContentType = "SINA大单", Page = html }); if (res is Dictionary <string, object> ) { var item = new { PageData = (res as Dictionary <string, object>)["PageData"], ContentType = "SINA大单", CreateTime = DateTime.Now, TradingDate = Convertor.CalTradingDate(DateTime.Now, "00:00:00"), PageIndex = i, PageCount = pageCount, }; LOGGER.Log(string.Format("SINA大单保存 {0} {1}", i, pageCount)); var arrayList = item.PageData as ArrayList; ///数据二维化,计算可以计算的,剩下的再想办法计算 for (int k = 0; k < arrayList.Count; k++) { totalCount++; var arrItem = arrayList[k] as Dictionary <string, object>; var svItem2D = new { StockCode = arrItem["StockCode"], StockName = arrItem["StockName"], TickTime = arrItem["交易时间"], Price = arrItem["成交价"], Turnover = arrItem["成交量"], PrevPrice = arrItem["之前价格"], Kind = arrItem["成交类型"], TradingTime = Convertor.CalTradingDate(item.TradingDate, arrItem["交易时间"].ToString()), TradingDate = item.TradingDate, CreateTime = item.CreateTime, PageIndex = i, PageCount = pageCount, RowIndex = k }; mongo.Save3(dbName, tempCollectionName, svItem2D); TaskStatusManager.Set(methodName, new { ID = methodName, PageIndex = i, PageCount = pageCount, Status = "已下载", CreateTime = DateTime.Now }); LOGGER.Log(string.Format("SINA大单2D保存 {0} {1} {2}", i, k, pageCount)); } } ThreadManager.Pause(seconds: 2); } TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now }); var countInDB = mongo.Count(CONST.DB.DBName_StockService, tempCollectionName, "{}"); if (totalCount == countInDB) { LOGGER.Log(string.Format("准备转移数据")); ///数据检查是否完毕,完备后插入 //DataStorage.MoveCollection(mongo, CONST.DB.DBName_StockService, tempCollectionName, "{}", mongo, CONST.DB.DBName_StockService, CONST.DB.CollectionName_DaDan, false); } else { LOGGER.Log(string.Format("数据对不上 {0} {1}", totalCount, countInDB)); } }