/// <summary>
/// 准备数据
/// </summary>
/// <returns></returns>
protected Queue <string> PrepareData(string methodName = null)
{
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var query = "{\"ContentType\":\"股票代码\",\"SortCode\":{$exists:true}}";
var sort = "{\"SortCode\":1}";
var resList = mongo.Find3("StockService", "BaseInfo", query, sort);
this.stockCodeDict = resList.ToDictionary(k => k["StockCode"].ToString(), v => v["StockName"].ToString());
var codeList = from item in resList orderby(int) item["SortCode"] select item["StockCode"].ToString();
var queue = CollectionTools.ToQueue <string>(codeList);
if (!string.IsNullOrWhiteSpace(methodName))
{
var status = TaskStatusManager.Get(methodName);
var from = string.Empty; ///上一次的起始位置
if (status.ContainsKey("StockCode"))
{
from = status["StockCode"].ToString();
}
while (!string.IsNullOrWhiteSpace(from) && 0 < queue.Count)
{
var stockCode = queue.Dequeue();
if (stockCode == from) ///若有状态,则从上次的位置开始下载
{
break;
}
}
}
return(queue);
}
/// <summary>
/// 下载大单数据
/// </summary>
public void GetDaDanData()
{
///获取页码码数
///下载每一页数据
///查找最近一个交易日的数据是否存在,若存在则不下载
///
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var webSource = DataSourceSINA.GetInstance();
var pageCount = 0;// webSource.GetDaDanPageCount();
if (0 == pageCount)
{
pageCount = int.Parse(WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceSINA&m=GetDaDanPageCount")));
}
var dbName = CONST.DB.DBName_StockService;
var collectionName = CONST.DB.CollectionName_DaDan;
///数据检查
var lastTradingDate = Convertor.CalTradingDate(DateTime.Now, "00:00:00");
///删除根据日期创建的临时表格
///将数据写入表格
///将这个数据转移到大表中
#region 存储临时数据
var tempCollectionName = collectionName + string.Format("{0:yyyyMMdd}", DateTime.Now);
mongo.Remove(dbName, tempCollectionName, "{}");
#endregion
var methodName = "GetDaDanData";
var status = TaskStatusManager.Get(methodName);
var startIndex = (status.ContainsKey("PageIndex")) ? int.Parse(status["PageIndex"].ToString()) : 0;
var totalCount = 0;
for (int i = startIndex; i < pageCount; i++)
{
var html = "";// webSource.GetDaDan(i);
if (string.IsNullOrWhiteSpace(html))
{
html = WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceSINA&m=GetDaDan&p={0}", i));
}
var res = NodeService.Get(CONST.NodeServiceUrl, "新浪", "GetDataFromHtml", new { ContentType = "SINA大单", Page = html });
if (res is Dictionary <string, object> )
{
var item = new
{
PageData = (res as Dictionary <string, object>)["PageData"],
ContentType = "SINA大单",
CreateTime = DateTime.Now,
TradingDate = Convertor.CalTradingDate(DateTime.Now, "00:00:00"),
PageIndex = i,
PageCount = pageCount,
};
LOGGER.Log(string.Format("SINA大单保存 {0} {1}", i, pageCount));
var arrayList = item.PageData as ArrayList;
///数据二维化，计算可以计算的，剩下的再想办法计算
for (int k = 0; k < arrayList.Count; k++)
{
totalCount++;
var arrItem = arrayList[k] as Dictionary <string, object>;
var svItem2D = new
{
StockCode = arrItem["StockCode"],
StockName = arrItem["StockName"],
TickTime = arrItem["交易时间"],
Price = arrItem["成交价"],
Turnover = arrItem["成交量"],
PrevPrice = arrItem["之前价格"],
Kind = arrItem["成交类型"],
TradingTime = Convertor.CalTradingDate(item.TradingDate, arrItem["交易时间"].ToString()),
TradingDate = item.TradingDate,
CreateTime = item.CreateTime,
PageIndex = i,
PageCount = pageCount,
RowIndex = k
};
mongo.Save3(dbName, tempCollectionName, svItem2D);
TaskStatusManager.Set(methodName, new { ID = methodName, PageIndex = i, PageCount = pageCount, Status = "已下载", CreateTime = DateTime.Now });
LOGGER.Log(string.Format("SINA大单2D保存 {0} {1} {2}", i, k, pageCount));
}
}
ThreadManager.Pause(seconds: 2);
}
TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now });
var countInDB = mongo.Count(CONST.DB.DBName_StockService, tempCollectionName, "{}");
if (totalCount == countInDB)
{
LOGGER.Log(string.Format("准备转移数据"));
///数据检查是否完毕,完备后插入
//DataStorage.MoveCollection(mongo, CONST.DB.DBName_StockService, tempCollectionName, "{}", mongo, CONST.DB.DBName_StockService, CONST.DB.CollectionName_DaDan, false);
}
else
{
LOGGER.Log(string.Format("数据对不上 {0} {1}", totalCount, countInDB));
}
}
