/// <summary> /// 准备数据 /// </summary> /// <returns></returns> protected Queue <string> PrepareData(string methodName = null) { var mongo = DataStorage.GetInstance(DBType.MongoDB); var query = "{\"ContentType\":\"股票代码\",\"SortCode\":{$exists:true}}"; var sort = "{\"SortCode\":1}"; var resList = mongo.Find3("StockService", "BaseInfo", query, sort); this.stockCodeDict = resList.ToDictionary(k => k["StockCode"].ToString(), v => v["StockName"].ToString()); var codeList = from item in resList orderby(int) item["SortCode"] select item["StockCode"].ToString(); var queue = CollectionTools.ToQueue <string>(codeList); if (!string.IsNullOrWhiteSpace(methodName)) { var status = TaskStatusManager.Get(methodName); var from = string.Empty; ///上一次的起始位置 if (status.ContainsKey("StockCode")) { from = status["StockCode"].ToString(); } while (!string.IsNullOrWhiteSpace(from) && 0 < queue.Count) { var stockCode = queue.Dequeue(); if (stockCode == from) ///若有状态,则从上次的位置开始下载 { break; } } } return(queue); }
/// <summary> /// 下载大单数据 /// </summary> public void GetDaDanData() { ///获取页码码数 ///下载每一页数据 ///查找最近一个交易日的数据是否存在,若存在则不下载 /// var mongo = DataStorage.GetInstance(DBType.MongoDB); var webSource = DataSourceSINA.GetInstance(); var pageCount = 0;// webSource.GetDaDanPageCount(); if (0 == pageCount) { pageCount = int.Parse(WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceSINA&m=GetDaDanPageCount"))); } var dbName = CONST.DB.DBName_StockService; var collectionName = CONST.DB.CollectionName_DaDan; ///数据检查 var lastTradingDate = Convertor.CalTradingDate(DateTime.Now, "00:00:00"); ///删除根据日期创建的临时表格 ///将数据写入表格 ///将这个数据转移到大表中 #region 存储临时数据 var tempCollectionName = collectionName + string.Format("{0:yyyyMMdd}", DateTime.Now); mongo.Remove(dbName, tempCollectionName, "{}"); #endregion var methodName = "GetDaDanData"; var status = TaskStatusManager.Get(methodName); var startIndex = (status.ContainsKey("PageIndex")) ? int.Parse(status["PageIndex"].ToString()) : 0; var totalCount = 0; for (int i = startIndex; i < pageCount; i++) { var html = "";// webSource.GetDaDan(i); if (string.IsNullOrWhiteSpace(html)) { html = WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceSINA&m=GetDaDan&p={0}", i)); } var res = NodeService.Get(CONST.NodeServiceUrl, "新浪", "GetDataFromHtml", new { ContentType = "SINA大单", Page = html }); if (res is Dictionary <string, object> ) { var item = new { PageData = (res as Dictionary <string, object>)["PageData"], ContentType = "SINA大单", CreateTime = DateTime.Now, TradingDate = Convertor.CalTradingDate(DateTime.Now, "00:00:00"), PageIndex = i, PageCount = pageCount, }; LOGGER.Log(string.Format("SINA大单保存 {0} {1}", i, pageCount)); var arrayList = item.PageData as ArrayList; ///数据二维化,计算可以计算的,剩下的再想办法计算 for (int k = 0; k < arrayList.Count; k++) { totalCount++; var arrItem = arrayList[k] as Dictionary <string, object>; var svItem2D = new { StockCode = arrItem["StockCode"], StockName = arrItem["StockName"], TickTime = arrItem["交易时间"], Price = arrItem["成交价"], Turnover = arrItem["成交量"], PrevPrice = arrItem["之前价格"], Kind = arrItem["成交类型"], TradingTime = Convertor.CalTradingDate(item.TradingDate, arrItem["交易时间"].ToString()), TradingDate = item.TradingDate, CreateTime = item.CreateTime, PageIndex = i, PageCount = pageCount, RowIndex = k }; mongo.Save3(dbName, tempCollectionName, svItem2D); TaskStatusManager.Set(methodName, new { ID = methodName, PageIndex = i, PageCount = pageCount, Status = "已下载", CreateTime = DateTime.Now }); LOGGER.Log(string.Format("SINA大单2D保存 {0} {1} {2}", i, k, pageCount)); } } ThreadManager.Pause(seconds: 2); } TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now }); var countInDB = mongo.Count(CONST.DB.DBName_StockService, tempCollectionName, "{}"); if (totalCount == countInDB) { LOGGER.Log(string.Format("准备转移数据")); ///数据检查是否完毕,完备后插入 //DataStorage.MoveCollection(mongo, CONST.DB.DBName_StockService, tempCollectionName, "{}", mongo, CONST.DB.DBName_StockService, CONST.DB.CollectionName_DaDan, false); } else { LOGGER.Log(string.Format("数据对不上 {0} {1}", totalCount, countInDB)); } }