/// <summary>
/// 加载数据
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private static void Loader_EventUrlEnqueue(object sender, EventArgs e)
{
Console.WriteLine("Loader_EventUrlEnqueue");
var res = mongo.Find("ths", "news", "{ Status:'NotDownloaded' }"); ///要下载的列表页面
if (CollectionTools.CanTraverse(res)) ///若允许遍历
{
foreach (var item in res)
{
if (null != item["Url"] && item["Url"].ToString().ToLower().StartsWith("http://")) ///若数据有效
{
loader.UrlEnqueue(item);
}
}
}
else
{
var root = new Dictionary <string, object>();
root["Url"] = "http://news.10jqka.com.cn/today_list/";
root["Status"] = "NotDownloaded";
root["Task"] = "ExtractLink"; ///尚未下载,需要分析链接
root["EntryPoint"] = "True"; ///是否是进入节点
loader.UrlEnqueue(root);
}
}
/// <summary>
/// 获取所有的股票代码 必须休市后再用 因为 股价变动造成排名变动
/// </summary>
public void GetALLStockCode()
{
///先检查本地或数据库是否有,若有就用,否则从网络获取
Console.WriteLine("初始化股票代码\t{0}", DateTime.Now);
if (0 == this.stockCodeDict.Count)
{
var list = mongo.Find(THS.CONST.DBName, THS.CONST.StockBaseInfo, "{}");
if (0 < list.Count) ///若找得到数据
{
this.stockCodeDict.Clear();
foreach (var item in list)
{
this.stockCodeDict.Add(item["StockCode"].ToString(), item["StockName"].ToString());
}
Console.WriteLine("股票基本信息从数据库初始化完毕");
}
else
{
Console.WriteLine("重新获取股票信息");
var headers = new Dictionary <string, string>();
headers.Add("Accept", "text/html,*/*; q=0.01");
headers.Add("Accept-Encoding", "gzip, deflate");
headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
headers.Add("Host", "q.10jqka.com.cn");
headers.Add("Referer", "http://q.10jqka.com.cn/");
headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36");
headers.Add("X-Requested-With", "XMLHttpRequest");
#region 获取分页数
var pageCount = 0;
var pagerUrl = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", 1);
var pagerHtml = http.GetGzip(pagerUrl, Encoding.GetEncoding("GBK"), headers);
if (100 < pagerHtml.Length)
{
pageCount = int.Parse(pagerHtml.Substring(pagerHtml.LastIndexOf("</span>") - 3, 3)); ///页码数
Console.WriteLine("股票代码页数{0}\t{1}", pageCount, DateTime.Now);
}
#endregion
var stringBuilder = new StringBuilder();
for (int i = 1; i <= pageCount; i++)
{
var url = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", i);
var html = http.GetGzip(url, Encoding.GetEncoding("GBK"), headers);
#region 股票代码处理
string[] trArray = html.Substring(html.IndexOf("<tbody>") + "<tbody>".Length).Split(new string[] { "<tr>", "</tr>" }, StringSplitOptions.RemoveEmptyEntries);
foreach (var tr in trArray)
{
var tdArray = tr.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries);
if (10 < tdArray.Length)
{
var stockCode = tdArray[3].Substring(tdArray[3].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", "");
var stockName = tdArray[5].Substring(tdArray[5].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", "");
if (!stockCodeDict.ContainsKey(stockCode) && !stockCodeDict.ContainsValue(stockName))
{
stockCodeDict.Add(stockCode, stockName);
}
else
{
throw new Exception("数据重复");
}
Console.WriteLine("正在添加 {0}\t{1}", stockCode, stockName);
}
}
#endregion
}
#region 输出检查
foreach (var item in stockCodeDict)
{
Console.WriteLine(item.Key + " " + item.Value.Replace(" ", string.Empty));
stringBuilder.AppendLine(item.Key + "\t" + item.Value.Replace(" ", string.Empty));
}
#endregion
#region 文件保存
File.WriteAllText(@"E:\stockCode.txt", stringBuilder.ToString(), Encoding.UTF8);
Console.WriteLine("stockCode.txt 保存完毕");
#endregion
#region 保存到数据库
foreach (var item in stockCodeDict)
{
mongo.Save(THS.CONST.DBName, THS.CONST.StockBaseInfo, new { StockCode = item.Key, StockName = item.Value, CreateTime = DateTime.Now, UpdateTime = DateTime.Now, TaskID = THS.CONST.TaskID });
}
#endregion
Console.WriteLine("股票代码添加完毕 {0}", DateTime.Now);
}
}
}
/// <summary>
/// 获取所有的股票代码 必须休市后再用 因为 股价变动造成排名变动
/// </summary>
public void GetALLStockCode()
{
///先检查本地或数据库是否有,若有就用,否则从网络获取
Console.WriteLine("初始化股票代码\t{0}", DateTime.Now);
if (0 == this.stockCodeDict.Count)
{
var list = mongo.Find(THS.CONST.DBName, THS.CONST.StockBaseInfo, "{}");
if (0 < list.Count) ///若找得到数据
{
this.stockCodeDict.Clear();
foreach (var item in list)
{
this.stockCodeDict.Add(item["StockCode"].ToString(), item["StockName"].ToString());
}
Console.WriteLine("股票基本信息从数据库初始化完毕");
}
else
{
Console.WriteLine("重新获取股票信息");
#region 获取分页数
var pageCount = 0;
var pagerUrl = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", 1);
var pagerHtml = http.GetGzip(pagerUrl);
if (100 < pagerHtml.Length)
{
pageCount = int.Parse(pagerHtml.Substring(pagerHtml.LastIndexOf("</span>") - 3, 3)); ///页码数
Console.WriteLine("股票代码页数{0}\t{1}", pageCount, DateTime.Now);
}
#endregion
var stringBuilder = new StringBuilder();
for (int i = 1; i <= pageCount; i++)
{
var url = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", i);
var html = http.GetGzip(url);
#region 股票代码处理
string[] trArray = html.Substring(html.IndexOf("<tbody>") + "<tbody>".Length).Split(new string[] { "<tr>", "</tr>" }, StringSplitOptions.RemoveEmptyEntries);
foreach (var tr in trArray)
{
var tdArray = tr.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries);
if (10 < tdArray.Length)
{
var stockCode = tdArray[3].Substring(tdArray[3].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", "");
var stockName = tdArray[5].Substring(tdArray[5].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", "");
if (!stockCodeDict.ContainsKey(stockCode) && !stockCodeDict.ContainsValue(stockName))
{
stockCodeDict.Add(stockCode, stockName);
}
else
{
throw new Exception("数据重复");
}
Console.WriteLine("正在添加 {0}\t{1}", stockCode, stockName);
}
}
#endregion
}
#region 输出检查
foreach (var item in stockCodeDict)
{
Console.WriteLine(item.Key + " " + item.Value.Replace(" ", string.Empty));
stringBuilder.AppendLine(item.Key + "\t" + item.Value.Replace(" ", string.Empty));
}
#endregion
#region 文件保存
File.WriteAllText(@"E:\stockCode.txt", stringBuilder.ToString(), Encoding.UTF8);
Console.WriteLine("stockCode.txt 保存完毕");
#endregion
#region 保存到数据库
foreach (var item in stockCodeDict)
{
mongo.Save(THS.CONST.DBName, THS.CONST.StockBaseInfo, new { StockCode = item.Key, StockName = item.Value, CreateTime = DateTime.Now, UpdateTime = DateTime.Now, TaskID = THS.CONST.TaskID });
}
#endregion
Console.WriteLine("股票代码添加完毕 {0}", DateTime.Now);
}
}
}
/// <summary>
/// KLIine
/// </summary>
/// <param name="mongodbConnectionString"></param>
/// <param name="mongodbDBName"></param>
/// <param name="mongodbCollectionName"></param>
/// <param name="sqlserverConnectionString"></param>
/// <param name="sqlserverDBName"></param>
/// <param name="sqlserverCollectionName"></param>
public void ConvertKLine2D(string mongodbConnectionString, string mongodbDBName, string mongodbCollectionName, string sqlserverConnectionString, string sqlserverDBName, string sqlserverCollectionName)
{
#region SQLServer 注册
SQLServer.Register("140", @"Data Source=192.168.0.140\sql2016;Initial Catalog=WJStock;Persist Security Info=True;User ID=sa;Password=111qqq!!!");
var mssql = SQLServer.GetInstance("140");
#endregion
#region MongoDB注册
MongoDB mongo = MongoDB.GetInst("mongodb://192.168.0.140:27017");
#endregion
var count = 0;
mongo.EventTraverse += (object sender, EventArgs e) =>
{
var ee = e as EventProcEventArgs;
var dict = ee.Default as Dictionary <string, object>;
var list = dict["Rows"] as Array;
var lscjmx = dict["历史成交明细"] as Array;
if (list is Array)
{
var itemArrayIndex = 0;
string sql = "INSERT INTO [DataDaDan2D] ([dbItemID] ,[StockCode] ,[StockName] ,[TradingDate] ,[Price] ,[Volume] ,[PrevPrice] ,[Kind],[ItemArrayIndex]) VALUES (@dbItemID ,@StockCode ,@StockName ,@TradingDate ,@Price ,@Volume ,@PrevPrice ,@Kind,@ItemArrayIndex)";
foreach (var item in list)
{
var srcItem = item as Dictionary <string, object>;
var svItem = new Dictionary <string, object>();
svItem["dbItemID"] = dict["_id"].ToString();
svItem["StockCode"] = dict["StockCode"];
svItem["StockName"] = dict["StockName"];
svItem["日期"] = Convert.ToDateTime(srcItem["日期"]);
svItem["开盘价"] = float.Parse(srcItem["开盘价"].ToString());
svItem["最高价"] = float.Parse(srcItem["最高价"].ToString());
svItem["收盘价"] = float.Parse(srcItem["收盘价"].ToString());
svItem["最低价"] = float.Parse(srcItem["最低价"].ToString());
svItem["交易量(股)"] = long.Parse(srcItem["交易量(股)"].ToString());
svItem["交易金额(元)"] = long.Parse(srcItem["交易金额(元)"].ToString());
svItem["ItemArrayIndex"] = itemArrayIndex++;
svItem["Source"] = "SINA日线";
mongo.Save("DataSource2D", "DataKLine2D", svItem);
//var paramList = new List<KeyValuePair<string, object>>();
//paramList.Add(new KeyValuePair<string, object>("@dbItemID", svItem["dbItemID"]));
//paramList.Add(new KeyValuePair<string, object>("@StockCode", svItem["StockCode"]));
//paramList.Add(new KeyValuePair<string, object>("@StockName", svItem["StockName"]));
//paramList.Add(new KeyValuePair<string, object>("@TradingDate", svItem["TradingDate"]));
//paramList.Add(new KeyValuePair<string, object>("@Price", svItem["Price"]));
//paramList.Add(new KeyValuePair<string, object>("@Volume", svItem["Volume"]));
//paramList.Add(new KeyValuePair<string, object>("@PrevPrice", svItem["PrevPrice"]));
//paramList.Add(new KeyValuePair<string, object>("@Kind", svItem["Kind"]));
//paramList.Add(new KeyValuePair<string, object>("@ItemArrayIndex", svItem["ItemArrayIndex"]));
//mssql.Save(sql, paramList);
}
}
if (lscjmx is Array)
{
var itemArrayIndex = 0;
foreach (var item in lscjmx)
{
var svItem = new Dictionary <string, object>();
svItem["dbItemID"] = dict["_id"].ToString();
svItem["StockCode"] = dict["StockCode"];
svItem["StockName"] = dict["StockName"];
svItem["ItemArrayIndex"] = itemArrayIndex++;
svItem["Source"] = "要下载的SINA日线历史成交明细页面";
svItem["下载URL"] = item;
svItem["父页面URL"] = dict["Url"];
svItem["当前状态"] = "未开始下载页面";
svItem["下一步任务"] = "下载页面";
svItem["作业创建日期"] = DateTime.Now;
mongo.Save("Jobs", "DownloadQueue", svItem);
}
}
Console.WriteLine("成功插入" + dict["_id"] + " " + (++count));
};
mongo.Find("DataSource", "DataKLine", "{}");
}
public async Task AddMediaAsync(MongoDB.Driver.IMongoCollection<CatalogObject> collection, CatalogObject json)
{
try
{
//var found = vho.TryGetValue(json.AltCode, out r);
var filter = Builders<CatalogObject>.Filter.Eq("AltCode", json.AltCode);
var found = await collection.Find(filter).ToListAsync();
if (found.Count > 0)
{
//check to see if its an exact copy, if it is then skip
var r = found[0];
if (r.GetHashCode() != json.GetHashCode())
{
if (json.IsHD)//only add values/updates from the HD version
{
//vho[json.AltCode] = json;
if (null != collection.Find(json.AltCode))
{
await collection.ReplaceOneAsync(x => x.AltCode == json.AltCode, json);
}
else
{
await collection.InsertOneAsync(json);
}
Compare(r, json);
}
else
{
//need to Add Avails data for nonHD here which requires making the private fields into a list of...
}
}
}
else
{
//Not this is unusual and an error
//So possible outcome here is we have same content but first four chars dont match, but content is same
//var dict = vho.Where(kvp => json.ProviderAssetID.Substring(4)).SelectMany(kvp => kvp.Value);
//var dict = vho.SelectMany(m => m).Where(k => vho.Keys.Contains(json.ProviderAssetID.Substring(4)));
/*var selectValues = (from keyValuePair in vho
where keyValuePair.Key.Contains(json.ProviderAssetID.Substring(4))
select keyValuePair.Value).ToList();
if (selectValues.Count > 1)
Console.WriteLine(vhoNumber + " : " + json.AltCode);
*/
//vho[json.AltCode] = json;
if (null == collection.Find(json.AltCode))
{
await collection.ReplaceOneAsync(x => x.AltCode == json.AltCode, json);
}
else
{
await collection.InsertOneAsync(json);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
