/// <summary>
/// 获取THS龙虎榜数据
/// </summary>
/// <param name="stockCode"></param>
/// <param name="source"></param>
/// <returns></returns>
public List <object> GetLHB(string stockCode, string source = "THS")
{
var resList = new List <object>();
if ("THS" == source.ToUpper() && !string.IsNullOrWhiteSpace(stockCode)) //
{
var lhbHtml = DataSourceTHS.CreateInstance().GetGGLHB(stockCode);
if (string.IsNullOrWhiteSpace(lhbHtml))
{
lhbHtml = WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceTHS&m=GetGGLHB&p={0}", stockCode));
}
var lhbDict = NodeService.Get(CONST.NodeServiceUrl, "同花顺", "GetDataFromHtml", new { ContentType = "THS个股龙虎榜", Page = lhbHtml });
resList.Add((lhbDict as Dictionary <string, object>)["PageData"]);
ThreadManager.Pause(seconds: 2);
var lhbmxUrlList = DataSourceTHS.CreateInstance().GetUrlGGLHBMX(lhbHtml);
foreach (var lhbMXUrl in lhbmxUrlList)
{
var lhbMXHtml = DataSourceTHS.CreateInstance().GetGGLHBMX(stockCode, lhbMXUrl);
if (string.IsNullOrWhiteSpace(lhbMXHtml))
{
//lhbHtml = WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceTHS&m=GetGGLHBMX&p={0}", stockCode));
}
var lhbMXDict = NodeService.Get(CONST.NodeServiceUrl, "同花顺", "GetDataFromHtml", new { ContentType = "THS个股龙虎榜明细", Page = lhbMXHtml });
resList.Add((lhbMXDict as Dictionary <string, object>)["PageData"]);
ThreadManager.Pause(seconds: 1);
}
}
return(resList);
}
/// <summary>
/// 获取SINA日线数据
/// </summary>
/// <param name="stockCode"></param>
/// <param name="year"></param>
/// <param name="jidu"></param>
/// <param name="source"></param>
/// <returns></returns>
public Dictionary <string, object> GetSINAKLineDay(string stockCode, int year, int jidu, string source = "SINA")
{
var html = DataSourceSINA.GetInstance().GetLSJY(stockCode, year, jidu);
var res = NodeService.Get(CONST.NodeServiceUrl, "新浪", "GetDataFromHtml", new { ContentType = "SINA历史交易", Page = html });//
var resDict = res as Dictionary <string, object>;
return(resDict["PageData"] as Dictionary <string, object>);
}
/// <summary>
/// 获取SINA财务摘要数据
/// </summary>
/// <param name="stockCode"></param>
/// <param name="source"></param>
/// <returns></returns>
public object GetCWZY(string stockCode, string source = "SINA")
{
var res = new object();
if ("SINA" == source.ToUpper() && !string.IsNullOrWhiteSpace(stockCode))// //WebDataSource.GetDataFromHttpAgent("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceSINA&m=GetCWZY&p=6018888");//
{
var html = DataSourceSINA.GetInstance().GetCWZY(stockCode);
var resDict = NodeService.Get(CONST.NodeServiceUrl, "新浪", "GetDataFromHtml", new { ContentType = "SINA财务摘要", Page = html });
return(resDict);
}
return(res);
}
/// <summary>
///
/// </summary>
/// <param name="stockCode"></param>
/// <param name="source"></param>
/// <returns></returns>
public ArrayList GetRZRQ(string stockCode, string source = "SINA")
{
var res = new ArrayList();
if ("SINA" == source.ToUpper() && !string.IsNullOrWhiteSpace(stockCode)) //
{
var rzrqHtml = DataSourceSINA.GetInstance().GetRZRQ(stockCode);
var rzrqDict = NodeService.Get(CONST.NodeServiceUrl, "新浪", "GetDataFromHtml", new { ContentType = "SINA融资融券", Page = rzrqHtml });
var pageData = (rzrqDict as Dictionary <string, object>)["PageData"] as Dictionary <string, object>;
return(pageData["Rows"] as ArrayList);
}
return(res);
}
/// <summary>
/// 获取THS资金流向
/// </summary>
/// <param name="stockCode"></param>
/// <param name="source"></param>
/// <returns></returns>
public object GetZJLX(string stockCode, string source = "THS")
{
var res = new object();
if ("THS" == source.ToUpper() && !string.IsNullOrWhiteSpace(stockCode)) //
{
var html = DataSourceTHS.CreateInstance().GetJZLX(stockCode);
if (string.IsNullOrWhiteSpace(html))
{
html = WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceTHS&m=GetZJLX&p={0}", stockCode));
}
var resDict = NodeService.Get(CONST.NodeServiceUrl, "同花顺", "GetDataFromHtml", new { ContentType = "THS资金流向", Page = html });
return(resDict);
}
return(res);
}
/// <summary>
/// 获取SINA股票雷达
/// </summary>
/// <returns></returns>
public List <Dictionary <string, object> > GetStockRadar()
{
var tradingDate = Convertor.CalTradingDate(DateTime.Now, "00:00:00");
var listHtml = DataSourceSINA.GetInstance().GetStockRadar();
var listData = new List <Dictionary <string, object> >();
foreach (var html in listHtml)
{
var resDict = NodeService.Get(CONST.NodeServiceUrl, "新浪", "GetDataFromHtml", new { ContentType = "SINA股市雷达", Page = html }) as Dictionary <string, object>;
var pageData = resDict["PageData"] as ArrayList;
if (pageData is ArrayList)
{
var itemList = pageData as ArrayList;
foreach (Dictionary <string, object> item in itemList)
{
var dict = item;
dict["异动时间"] = Convertor.CalTradingDate(DateTime.Now, item["异动时间"].ToString());
listData.Add(dict);
}
}
}
return(listData);
}
/// <summary>
/// 获取指定日期的新闻
/// </summary>
public void GetNewsListCJYW(string dateTime)
{
if (StringChecker.IsDateTime(dateTime))
{
var formatDate = string.Format("{0:yyyyMMdd}", Convert.ToDateTime(dateTime));
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var webSrc = DataSourceTHS.CreateInstance();
var newNewsList = new Dictionary <string, object>();
var listHtml = webSrc.GetNewsListCJYW(Convert.ToDateTime(dateTime));
var dbName = CONST.DB.DBName_StockService;
var collectionNews = CONST.DB.CollectionName_News;
if (!string.IsNullOrWhiteSpace(listHtml))
{
var resDict = NodeService.Get(CONST.NodeServiceUrl, "同花顺", "GetDataFromHtml", new { ContentType = "THS财经要闻新闻列表", Page = listHtml }) as Dictionary <string, object>;
var resList = resDict["PageData"] as ArrayList;
if (null != resList)
{
foreach (Dictionary <string, object> listItem in resList)
{
var href = listItem["Href"].ToString().Trim();
var parentUrl = string.Format("http://news.10jqka.com.cn/today_list/{0}/", formatDate).Trim();
var newsHtml = webSrc.GetNewsArticle(href, parentUrl);
if (!string.IsNullOrWhiteSpace(newsHtml))
{
TaskStatusManager.Set("GetNewsListCJYW", new { ID = "GetNewsListCJYW", CreateTime = DateTime.Now, Status = "准备操作THS财经要闻新闻详细", Html = newsHtml });
var resItem = NodeService.Get(CONST.NodeServiceUrl, "同花顺", "GetDataFromHtml", new { ContentType = "THS财经要闻新闻详细", Page = newsHtml }) as Dictionary <string, object>;
var resDetail = resItem["PageData"] as Dictionary <string, object>;
if (null != resDetail)
{
var newsCreateTimeString = resDetail["CreateTime"].ToString().Trim();
var svItem = new
{
ContentType = "THS财经要闻",
Title = resDetail["Title"].ToString().Trim(),
Url = href,
ParentUrl = parentUrl,
SourceHref = resDetail["SourceHref"].ToString().Trim(),
SourceName = resDetail["SourceName"].ToString().Trim(),
NewsCreateTime = (StringChecker.IsDateTime(newsCreateTimeString))?Convert.ToDateTime(newsCreateTimeString):DateTime.MinValue,
Content = resDetail["Content"].ToString().Trim(),
Tag = Convert.ToInt32(formatDate),
CreateTime = DateTime.Now,
PageMD5 = "无"
};
newNewsList.Add(href, svItem);
LOGGER.Log(string.Format("获取一个新闻正文 {0}", svItem.Title));
ThreadManager.Pause(seconds: 5); ///5秒钟更新一次新闻
///分词
var task = Task.Factory.StartNew(() =>
{
var fcZStartTime = DateTime.Now;
this.SaveFenCi(svItem.Content, svItem.Url);
LOGGER.Log(string.Format("分词花费时间 开始时间：{0} 花费时间：{1}", fcZStartTime, DateTime.Now - fcZStartTime));
});
}
}
}
foreach (var svItem in newNewsList)
{
var filter = "{\"Url\":\"" + svItem.Key + "\"}";
mongo.Save3(dbName, collectionNews, svItem.Value, filter);
}
}
}
else
{
LOGGER.Log(string.Format("获取的新闻列表为空白"));
}
}
else
{
LOGGER.Log(string.Format("传入的时间字符串不对:{0}", dateTime));
ThreadManager.Pause(minutes: 5);
}
}
/// <summary>
/// 下载大单数据
/// </summary>
public void GetDaDanData()
{
///获取页码码数
///下载每一页数据
///查找最近一个交易日的数据是否存在,若存在则不下载
///
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var webSource = DataSourceSINA.GetInstance();
var pageCount = 0;// webSource.GetDaDanPageCount();
if (0 == pageCount)
{
pageCount = int.Parse(WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceSINA&m=GetDaDanPageCount")));
}
var dbName = CONST.DB.DBName_StockService;
var collectionName = CONST.DB.CollectionName_DaDan;
///数据检查
var lastTradingDate = Convertor.CalTradingDate(DateTime.Now, "00:00:00");
///删除根据日期创建的临时表格
///将数据写入表格
///将这个数据转移到大表中
#region 存储临时数据
var tempCollectionName = collectionName + string.Format("{0:yyyyMMdd}", DateTime.Now);
mongo.Remove(dbName, tempCollectionName, "{}");
#endregion
var methodName = "GetDaDanData";
var status = TaskStatusManager.Get(methodName);
var startIndex = (status.ContainsKey("PageIndex")) ? int.Parse(status["PageIndex"].ToString()) : 0;
var totalCount = 0;
for (int i = startIndex; i < pageCount; i++)
{
var html = "";// webSource.GetDaDan(i);
if (string.IsNullOrWhiteSpace(html))
{
html = WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceSINA&m=GetDaDan&p={0}", i));
}
var res = NodeService.Get(CONST.NodeServiceUrl, "新浪", "GetDataFromHtml", new { ContentType = "SINA大单", Page = html });
if (res is Dictionary <string, object> )
{
var item = new
{
PageData = (res as Dictionary <string, object>)["PageData"],
ContentType = "SINA大单",
CreateTime = DateTime.Now,
TradingDate = Convertor.CalTradingDate(DateTime.Now, "00:00:00"),
PageIndex = i,
PageCount = pageCount,
};
LOGGER.Log(string.Format("SINA大单保存 {0} {1}", i, pageCount));
var arrayList = item.PageData as ArrayList;
///数据二维化，计算可以计算的，剩下的再想办法计算
for (int k = 0; k < arrayList.Count; k++)
{
totalCount++;
var arrItem = arrayList[k] as Dictionary <string, object>;
var svItem2D = new
{
StockCode = arrItem["StockCode"],
StockName = arrItem["StockName"],
TickTime = arrItem["交易时间"],
Price = arrItem["成交价"],
Turnover = arrItem["成交量"],
PrevPrice = arrItem["之前价格"],
Kind = arrItem["成交类型"],
TradingTime = Convertor.CalTradingDate(item.TradingDate, arrItem["交易时间"].ToString()),
TradingDate = item.TradingDate,
CreateTime = item.CreateTime,
PageIndex = i,
PageCount = pageCount,
RowIndex = k
};
mongo.Save3(dbName, tempCollectionName, svItem2D);
TaskStatusManager.Set(methodName, new { ID = methodName, PageIndex = i, PageCount = pageCount, Status = "已下载", CreateTime = DateTime.Now });
LOGGER.Log(string.Format("SINA大单2D保存 {0} {1} {2}", i, k, pageCount));
}
}
ThreadManager.Pause(seconds: 2);
}
TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now });
var countInDB = mongo.Count(CONST.DB.DBName_StockService, tempCollectionName, "{}");
if (totalCount == countInDB)
{
LOGGER.Log(string.Format("准备转移数据"));
///数据检查是否完毕,完备后插入
//DataStorage.MoveCollection(mongo, CONST.DB.DBName_StockService, tempCollectionName, "{}", mongo, CONST.DB.DBName_StockService, CONST.DB.CollectionName_DaDan, false);
}
else
{
LOGGER.Log(string.Format("数据对不上 {0} {1}", totalCount, countInDB));
}
}
public Node Get(int id)
{
return(Service.Get(id));
}
