/// <summary> /// 加载数据 /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private static void Loader_EventUrlEnqueue(object sender, EventArgs e) { Console.WriteLine("Loader_EventUrlEnqueue"); var res = mongo.Find("ths", "news", "{ Status:'NotDownloaded' }"); ///要下载的列表页面 if (CollectionTools.CanTraverse(res)) ///若允许遍历 { foreach (var item in res) { if (null != item["Url"] && item["Url"].ToString().ToLower().StartsWith("http://")) ///若数据有效 { loader.UrlEnqueue(item); } } } else { var root = new Dictionary <string, object>(); root["Url"] = "http://news.10jqka.com.cn/today_list/"; root["Status"] = "NotDownloaded"; root["Task"] = "ExtractLink"; ///尚未下载,需要分析链接 root["EntryPoint"] = "True"; ///是否是进入节点 loader.UrlEnqueue(root); } }
/// <summary> /// 获取所有的股票代码 必须休市后再用 因为 股价变动造成排名变动 /// </summary> public void GetALLStockCode() { ///先检查本地或数据库是否有,若有就用,否则从网络获取 Console.WriteLine("初始化股票代码\t{0}", DateTime.Now); if (0 == this.stockCodeDict.Count) { var list = mongo.Find(THS.CONST.DBName, THS.CONST.StockBaseInfo, "{}"); if (0 < list.Count) ///若找得到数据 { this.stockCodeDict.Clear(); foreach (var item in list) { this.stockCodeDict.Add(item["StockCode"].ToString(), item["StockName"].ToString()); } Console.WriteLine("股票基本信息从数据库初始化完毕"); } else { Console.WriteLine("重新获取股票信息"); var headers = new Dictionary <string, string>(); headers.Add("Accept", "text/html,*/*; q=0.01"); headers.Add("Accept-Encoding", "gzip, deflate"); headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"); headers.Add("Host", "q.10jqka.com.cn"); headers.Add("Referer", "http://q.10jqka.com.cn/"); headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); headers.Add("X-Requested-With", "XMLHttpRequest"); #region 获取分页数 var pageCount = 0; var pagerUrl = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", 1); var pagerHtml = http.GetGzip(pagerUrl, Encoding.GetEncoding("GBK"), headers); if (100 < pagerHtml.Length) { pageCount = int.Parse(pagerHtml.Substring(pagerHtml.LastIndexOf("</span>") - 3, 3)); ///页码数 Console.WriteLine("股票代码页数{0}\t{1}", pageCount, DateTime.Now); } #endregion var stringBuilder = new StringBuilder(); for (int i = 1; i <= pageCount; i++) { var url = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", i); var html = http.GetGzip(url, Encoding.GetEncoding("GBK"), headers); #region 股票代码处理 string[] trArray = html.Substring(html.IndexOf("<tbody>") + "<tbody>".Length).Split(new string[] { "<tr>", "</tr>" }, StringSplitOptions.RemoveEmptyEntries); foreach (var tr in trArray) { var tdArray = tr.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries); if (10 < tdArray.Length) { var stockCode = tdArray[3].Substring(tdArray[3].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", ""); var stockName = tdArray[5].Substring(tdArray[5].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", ""); if (!stockCodeDict.ContainsKey(stockCode) && !stockCodeDict.ContainsValue(stockName)) { stockCodeDict.Add(stockCode, stockName); } else { throw new Exception("数据重复"); } Console.WriteLine("正在添加 {0}\t{1}", stockCode, stockName); } } #endregion } #region 输出检查 foreach (var item in stockCodeDict) { Console.WriteLine(item.Key + " " + item.Value.Replace(" ", string.Empty)); stringBuilder.AppendLine(item.Key + "\t" + item.Value.Replace(" ", string.Empty)); } #endregion #region 文件保存 File.WriteAllText(@"E:\stockCode.txt", stringBuilder.ToString(), Encoding.UTF8); Console.WriteLine("stockCode.txt 保存完毕"); #endregion #region 保存到数据库 foreach (var item in stockCodeDict) { mongo.Save(THS.CONST.DBName, THS.CONST.StockBaseInfo, new { StockCode = item.Key, StockName = item.Value, CreateTime = DateTime.Now, UpdateTime = DateTime.Now, TaskID = THS.CONST.TaskID }); } #endregion Console.WriteLine("股票代码添加完毕 {0}", DateTime.Now); } } }
/// <summary> /// 获取所有的股票代码 必须休市后再用 因为 股价变动造成排名变动 /// </summary> public void GetALLStockCode() { ///先检查本地或数据库是否有,若有就用,否则从网络获取 Console.WriteLine("初始化股票代码\t{0}", DateTime.Now); if (0 == this.stockCodeDict.Count) { var list = mongo.Find(THS.CONST.DBName, THS.CONST.StockBaseInfo, "{}"); if (0 < list.Count) ///若找得到数据 { this.stockCodeDict.Clear(); foreach (var item in list) { this.stockCodeDict.Add(item["StockCode"].ToString(), item["StockName"].ToString()); } Console.WriteLine("股票基本信息从数据库初始化完毕"); } else { Console.WriteLine("重新获取股票信息"); #region 获取分页数 var pageCount = 0; var pagerUrl = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", 1); var pagerHtml = http.GetGzip(pagerUrl); if (100 < pagerHtml.Length) { pageCount = int.Parse(pagerHtml.Substring(pagerHtml.LastIndexOf("</span>") - 3, 3)); ///页码数 Console.WriteLine("股票代码页数{0}\t{1}", pageCount, DateTime.Now); } #endregion var stringBuilder = new StringBuilder(); for (int i = 1; i <= pageCount; i++) { var url = string.Format(@"http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/{0}/ajax/1/", i); var html = http.GetGzip(url); #region 股票代码处理 string[] trArray = html.Substring(html.IndexOf("<tbody>") + "<tbody>".Length).Split(new string[] { "<tr>", "</tr>" }, StringSplitOptions.RemoveEmptyEntries); foreach (var tr in trArray) { var tdArray = tr.Split(new string[] { "<td>", "</td>" }, StringSplitOptions.RemoveEmptyEntries); if (10 < tdArray.Length) { var stockCode = tdArray[3].Substring(tdArray[3].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", ""); var stockName = tdArray[5].Substring(tdArray[5].IndexOf("target=\"_blank\">") + "target=\"_blank\">".Length).Replace("</a>", string.Empty).Replace(" ", ""); if (!stockCodeDict.ContainsKey(stockCode) && !stockCodeDict.ContainsValue(stockName)) { stockCodeDict.Add(stockCode, stockName); } else { throw new Exception("数据重复"); } Console.WriteLine("正在添加 {0}\t{1}", stockCode, stockName); } } #endregion } #region 输出检查 foreach (var item in stockCodeDict) { Console.WriteLine(item.Key + " " + item.Value.Replace(" ", string.Empty)); stringBuilder.AppendLine(item.Key + "\t" + item.Value.Replace(" ", string.Empty)); } #endregion #region 文件保存 File.WriteAllText(@"E:\stockCode.txt", stringBuilder.ToString(), Encoding.UTF8); Console.WriteLine("stockCode.txt 保存完毕"); #endregion #region 保存到数据库 foreach (var item in stockCodeDict) { mongo.Save(THS.CONST.DBName, THS.CONST.StockBaseInfo, new { StockCode = item.Key, StockName = item.Value, CreateTime = DateTime.Now, UpdateTime = DateTime.Now, TaskID = THS.CONST.TaskID }); } #endregion Console.WriteLine("股票代码添加完毕 {0}", DateTime.Now); } } }
/// <summary> /// KLIine /// </summary> /// <param name="mongodbConnectionString"></param> /// <param name="mongodbDBName"></param> /// <param name="mongodbCollectionName"></param> /// <param name="sqlserverConnectionString"></param> /// <param name="sqlserverDBName"></param> /// <param name="sqlserverCollectionName"></param> public void ConvertKLine2D(string mongodbConnectionString, string mongodbDBName, string mongodbCollectionName, string sqlserverConnectionString, string sqlserverDBName, string sqlserverCollectionName) { #region SQLServer 注册 SQLServer.Register("140", @"Data Source=192.168.0.140\sql2016;Initial Catalog=WJStock;Persist Security Info=True;User ID=sa;Password=111qqq!!!"); var mssql = SQLServer.GetInstance("140"); #endregion #region MongoDB注册 MongoDB mongo = MongoDB.GetInst("mongodb://192.168.0.140:27017"); #endregion var count = 0; mongo.EventTraverse += (object sender, EventArgs e) => { var ee = e as EventProcEventArgs; var dict = ee.Default as Dictionary <string, object>; var list = dict["Rows"] as Array; var lscjmx = dict["历史成交明细"] as Array; if (list is Array) { var itemArrayIndex = 0; string sql = "INSERT INTO [DataDaDan2D] ([dbItemID] ,[StockCode] ,[StockName] ,[TradingDate] ,[Price] ,[Volume] ,[PrevPrice] ,[Kind],[ItemArrayIndex]) VALUES (@dbItemID ,@StockCode ,@StockName ,@TradingDate ,@Price ,@Volume ,@PrevPrice ,@Kind,@ItemArrayIndex)"; foreach (var item in list) { var srcItem = item as Dictionary <string, object>; var svItem = new Dictionary <string, object>(); svItem["dbItemID"] = dict["_id"].ToString(); svItem["StockCode"] = dict["StockCode"]; svItem["StockName"] = dict["StockName"]; svItem["日期"] = Convert.ToDateTime(srcItem["日期"]); svItem["开盘价"] = float.Parse(srcItem["开盘价"].ToString()); svItem["最高价"] = float.Parse(srcItem["最高价"].ToString()); svItem["收盘价"] = float.Parse(srcItem["收盘价"].ToString()); svItem["最低价"] = float.Parse(srcItem["最低价"].ToString()); svItem["交易量(股)"] = long.Parse(srcItem["交易量(股)"].ToString()); svItem["交易金额(元)"] = long.Parse(srcItem["交易金额(元)"].ToString()); svItem["ItemArrayIndex"] = itemArrayIndex++; svItem["Source"] = "SINA日线"; mongo.Save("DataSource2D", "DataKLine2D", svItem); //var paramList = new List<KeyValuePair<string, object>>(); //paramList.Add(new KeyValuePair<string, object>("@dbItemID", svItem["dbItemID"])); //paramList.Add(new KeyValuePair<string, object>("@StockCode", svItem["StockCode"])); //paramList.Add(new KeyValuePair<string, object>("@StockName", svItem["StockName"])); //paramList.Add(new KeyValuePair<string, object>("@TradingDate", svItem["TradingDate"])); //paramList.Add(new KeyValuePair<string, object>("@Price", svItem["Price"])); //paramList.Add(new KeyValuePair<string, object>("@Volume", svItem["Volume"])); //paramList.Add(new KeyValuePair<string, object>("@PrevPrice", svItem["PrevPrice"])); //paramList.Add(new KeyValuePair<string, object>("@Kind", svItem["Kind"])); //paramList.Add(new KeyValuePair<string, object>("@ItemArrayIndex", svItem["ItemArrayIndex"])); //mssql.Save(sql, paramList); } } if (lscjmx is Array) { var itemArrayIndex = 0; foreach (var item in lscjmx) { var svItem = new Dictionary <string, object>(); svItem["dbItemID"] = dict["_id"].ToString(); svItem["StockCode"] = dict["StockCode"]; svItem["StockName"] = dict["StockName"]; svItem["ItemArrayIndex"] = itemArrayIndex++; svItem["Source"] = "要下载的SINA日线历史成交明细页面"; svItem["下载URL"] = item; svItem["父页面URL"] = dict["Url"]; svItem["当前状态"] = "未开始下载页面"; svItem["下一步任务"] = "下载页面"; svItem["作业创建日期"] = DateTime.Now; mongo.Save("Jobs", "DownloadQueue", svItem); } } Console.WriteLine("成功插入" + dict["_id"] + " " + (++count)); }; mongo.Find("DataSource", "DataKLine", "{}"); }
public async Task AddMediaAsync(MongoDB.Driver.IMongoCollection<CatalogObject> collection, CatalogObject json) { try { //var found = vho.TryGetValue(json.AltCode, out r); var filter = Builders<CatalogObject>.Filter.Eq("AltCode", json.AltCode); var found = await collection.Find(filter).ToListAsync(); if (found.Count > 0) { //check to see if its an exact copy, if it is then skip var r = found[0]; if (r.GetHashCode() != json.GetHashCode()) { if (json.IsHD)//only add values/updates from the HD version { //vho[json.AltCode] = json; if (null != collection.Find(json.AltCode)) { await collection.ReplaceOneAsync(x => x.AltCode == json.AltCode, json); } else { await collection.InsertOneAsync(json); } Compare(r, json); } else { //need to Add Avails data for nonHD here which requires making the private fields into a list of... } } } else { //Not this is unusual and an error //So possible outcome here is we have same content but first four chars dont match, but content is same //var dict = vho.Where(kvp => json.ProviderAssetID.Substring(4)).SelectMany(kvp => kvp.Value); //var dict = vho.SelectMany(m => m).Where(k => vho.Keys.Contains(json.ProviderAssetID.Substring(4))); /*var selectValues = (from keyValuePair in vho where keyValuePair.Key.Contains(json.ProviderAssetID.Substring(4)) select keyValuePair.Value).ToList(); if (selectValues.Count > 1) Console.WriteLine(vhoNumber + " : " + json.AltCode); */ //vho[json.AltCode] = json; if (null == collection.Find(json.AltCode)) { await collection.ReplaceOneAsync(x => x.AltCode == json.AltCode, json); } else { await collection.InsertOneAsync(json); } } } catch (Exception ex) { Console.WriteLine(ex.Message); } }