/// <summary> /// 初始化大商所仓单数据 /// </summary> public static void GetDSFDataRepository_First() { var ibll = OperationContext.BLLSession; string[] years = new string[] { "2018" }; int[] months = new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; string[] days = new string[] { "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12" , "13", "14", "15", "16", "17", "18", "19", "20", "21", "22" , "23", "24", "25", "26", "27", "28", "29", "30", "31" }; foreach (string year in years) { foreach (int month in months) { foreach (string day in days) { int d = Convert.ToInt32(string.Format("{0}{1}{2}", year, month + 1 < 10 ? "0" + (month + 1).ToString() : (month + 1).ToString(), day)); if (d < 20180616) { continue; } if (ibll.FDataReposInit.where (a => a.Date == d && a.TradeHouse == TradeHouseType.dce.ToString() && a.Type == InitContentType.Cangdan.ToString()).Count() < 1) { string url = string.Format("http://www.dce.com.cn/publicweb/quotesdata/wbillWeeklyQuotes.html?year={0}&month={1}&day={2}", year, month, day); var model = CrawlerUtils.GetDSFDataRepository_First(url); ibll.FDataReposInit.Add(model); int n = ibll.FDataReposInit.SaveChanges(); System.Console.WriteLine("大商:" + d); } } } } }
public static void GetDSFDataRepository_Second() { var ibll = OperationContext.BLLSession; int count = ibll.FDataReposInit.where (a => a.ID > 0 && a.TradeHouse == TradeHouseType.dce.ToString() && a.IsCheckFinish == 0).Count(); for (int page = 1; page <= (count / 10) + 1; page++) { List <FDataReposInit> list = ibll.FDataReposInit.where (a => a.ID > 0 && a.TradeHouse == TradeHouseType.dce.ToString() && a.IsCheckFinish == 0).OrderBy(s => s.ID).Take(10).ToList(); foreach (FDataReposInit model in list) { if (ibll.FDataRepository.where (a => a.Date == model.Date && a.TradeHouse == TradeHouseType.dce.ToString()).Count() < 1) { List <FDataRepository> resplist = CrawlerUtils.GetDSFDataRepository_Second(model.Content, model.Date); foreach (var item in resplist) { ibll.FDataRepository.Add(item); } model.IsCheckFinish = 1; ibll.FDataReposInit.Update(model, new string[] { "IsCheckFinish" }); int num = ibll.SaveChanges(); System.Console.WriteLine("大商Check:" + model.Date); } else { continue; } } } System.Console.WriteLine("大商Check:FINISH"); }
/// <summary> /// 上海仓单初始化 /// 2011-20140516 /// </summary> public static void GetSHFDataRepository_FirstD() { TaskStart: try { var ibll = OperationContext.BLLSession; int count = ibll.FDataReposInit.where (a => a.Date <= 20140516 && a.TradeHouse == TradeHouseType.shfe.ToString() && a.IsCheckFinish == 0).Count(); for (int page = 1; page <= (count / 10) + 1; page++) { List <FDataReposInit> list = ibll.FDataReposInit.where (a => a.Date <= 20140516 && a.TradeHouse == TradeHouseType.shfe.ToString() && a.IsCheckFinish == 0).OrderBy(s => s.ID).Take(10).ToList(); foreach (FDataReposInit model in list) { List <FDataRepository> resplist = CrawlerUtils.GetSHFDataRepository_FirstD(model.Content, model.Date); foreach (var item in resplist) { ibll.FDataRepository.Add(item); } model.IsCheckFinish = 1; ibll.FDataReposInit.Update(model, new string[] { "IsCheckFinish" }); int num = ibll.SaveChanges(); System.Console.WriteLine("上海2011-2014Check:" + model.Date); } } System.Console.WriteLine("上海2011-2014Check:FINISH"); } catch (Exception ex) { System.Console.WriteLine(string.Format("异常:{0} 开始休眠", ex.Message)); System.Threading.Thread.Sleep(60000); goto TaskStart; } }
/// <summary> /// 获取商品交易所仓单数据 /// 2011- /// </summary> public static void GetZZFDataRepository_First() { ZZTaskStart: try { var ibll = OperationContext.BLLSession; //string[] years = new string[] { "2011", "2012", "2013", "2014", "2015", "2016", "2017", "2018" }; string[] years = new string[] { "2018" }; int[] months = new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; string[] days = new string[] { "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12" , "13", "14", "15", "16", "17", "18", "19", "20", "21", "22" , "23", "24", "25", "26", "27", "28", "29", "30", "31" }; foreach (string year in years) { foreach (int month in months) { foreach (string day in days) { int date = Convert.ToInt32(string.Format("{0}{1}{2}", year, month + 1 < 10 ? "0" + (month + 1).ToString() : (month + 1).ToString(), day)); if (date < 20180616) { continue; } if (ibll.FDataReposInit.where (a => a.Date == date && a.TradeHouse == TradeHouseType.czce.ToString() && a.Type == InitContentType.Cangdan.ToString()).Count() < 1) { string url = ""; if (date < 20151001) { url = string.Format("http://www.czce.com.cn/portal/exchange/{0}/datawhsheet/{1}.htm", year, date); } else if (date < 20180616) { url = string.Format("http://www.czce.com.cn/portal/DFSStaticFiles/Future/{0}/{1}/FutureDataWhsheet.htm", year, date); } else { url = string.Format("http://www.czce.com.cn/cn/DFSStaticFiles/Future/{0}/{1}/FutureDataWhsheet.htm", year, date); } var model = CrawlerUtils.GetZZFDataRepository_First(url, date); if (model != null) { ibll.FDataReposInit.Add(model); System.Console.WriteLine("郑州init:" + date); } } } int n = ibll.FDataReposInit.SaveChanges(); System.Console.WriteLine("郑州:Save --{0}", n); } } System.Console.WriteLine("郑州:FINISH"); } catch (Exception ex) { System.Console.WriteLine("郑商所仓单:{0}", ex.Message); System.Threading.Thread.Sleep(60000); goto ZZTaskStart; } }
/// <summary> /// 20140519-2018 上海仓单数据 /// </summary> public static void GetSHFDataRepository_Second() { var ibll = OperationContext.BLLSession; string[] years = new string[] { "2018" }; int[] months = new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; string[] days = new string[] { "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12" , "13", "14", "15", "16", "17", "18", "19", "20", "21", "22" , "23", "24", "25", "26", "27", "28", "29", "30", "31" }; foreach (string year in years) { foreach (int month in months) { foreach (string day in days) { int date = Convert.ToInt32(string.Format("{0}{1}{2}", year, month + 1 < 10 ? "0" + (month + 1).ToString() : (month + 1).ToString(), day)); if (date <= 20180616) { continue; } if (ibll.FDataReposInit.where (a => a.Date == date && a.TradeHouse == TradeHouseType.shfe.ToString() && a.Type == InitContentType.Cangdan.ToString()).Count() < 1) { string url = string.Format("http://www.shfe.com.cn/data/dailydata/{0}dailystock.dat", date); var model = CrawlerUtils.GetSHFDataRepository_Second(url, date.ToString()); if (model != null) { ibll.FDataReposInit.Add(model); System.Console.WriteLine("上海:" + date); } } } int n = ibll.FDataReposInit.SaveChanges(); System.Console.WriteLine("上海:Save --{0}", n); } } System.Console.WriteLine("上海:FINISH"); }
/// <summary> /// /// </summary> /// <param name="args"></param> static void Main(string[] args) { //string str = "{\"VARNAME\":\"铜$$COPPER\",\"VARSORT\":0,\"REGNAME\":\"上海$$Shanghai\",\"REGSORT\":0,\"WHABBRNAME\":\"期晟公司$$Qisheng\",\"WHROWS\":12,\"WGHTUNIT\":\"2\",\"WRTWGHTS\":1919,\"WRTCHANGE\":0,\"ROWORDER\":1,\"ROWSTATUS\":\"0\"}"; //string s = Regex.Replace(str, "(,\"VARSORT\".+?,)", ","); //s = Regex.Replace(s, "(,\"REGSORT\".+?,)", ","); //s = Regex.Replace(s, "(,\"WHROWS\".+?,)", ","); //System.Console.WriteLine(s); //s = Regex.Replace(s, "(,\"ROWORDER\".+?,)", ","); //s = Regex.Replace(s, "(,\"WGHTUNIT\".+?,)", ","); //s = Regex.Replace(s, "(,\"ROWSTATUS\".+?})", "}"); //s = Regex.Replace(s, "(\\$\\$.+?\")", "\""); //System.Console.WriteLine(s); //string title = "品种:苹果AP单位:张日期:2018-06-19"; //string temp = Regex.Match(title, "(品种:.+单位:)").ToString(); //System.Console.WriteLine(temp); //temp = temp.Replace("品种:", "").Replace("单位:", ""); //System.Console.WriteLine(temp); //System.Console.WriteLine("---------------------------------------------------------------"); //System.Console.ReadKey(); //var ibll = OperationContext.BLLSession; //string str = "/dalianshangpin/yw/fw/jystz/ywtz/6095044/index.html"; //string no = Regex.Match(str, "(/[0-9]+/)").ToString(); //System.Console.WriteLine(no); #region NewsTest //var list = new List<FNews> { }; //var pageIndex = 1; //while (pageIndex > 0 && pageIndex <= 2) //{ // /* 郑州所 这个站点做了处理 // * 1.登录会话做了限制,会被重定向 // * 2.详情被生成为PDF格式,需要解析 // * */ // var news = CrawlerUtils.GetNewsFromUrl_ZZ(1, pageIndex); // foreach (var item in news) // { // //ibll.FNews.Add(item); // System.Console.WriteLine($"{item.AddDate.Value.ToShortDateString()} {item.NewsTitle}~({(item.NewContent.Length > 0 ? "有采集到详情" : "无")})"); // } // /* // int n = ibll.FNews.SaveChanges(); // if (n > 0) // { // System.Console.WriteLine("新闻:save-{0}", n); // } // */ // if (news.Count == 0) // { // pageIndex = 0; // break; // } // // 请求太快貌似会被4O4 // Thread.Sleep(1000); // pageIndex++; //} //System.Console.WriteLine("新闻:Finish"); #endregion //SX_SiteConfig site = new SX_SiteConfig() { }; //site.SiteHost = "http://www.chancheng.gov.cn"; //site.SiteName = "佛山市禅城区教育局"; //site.SiteUrl = "http://www.chancheng.gov.cn/ccjy/060102/bmlist2.shtml"; //site.SiteListXPath = "/html/body/div[1]/div[2]/div/div[2]/div[2]"; //site.SiteItemXPath = "div/div"; //var list=CrawlerUtils.FoShanChanChengedu(site); SX_SiteConfig site = new SX_SiteConfig() { }; site.SiteHost = "http://www.longmen.gov.cn"; site.SiteName = "山香教育惠州分校"; site.SiteUrl = "http://www.shanweirc.com/more.jsp?sortid=1701&pagex=1"; site.SiteListXPath = "//*[@id=\"person-news\"]"; site.SiteItemXPath = "li"; var list = CrawlerUtils.SWChengQu(site); System.Console.WriteLine(list.Count); System.Console.ReadKey(); }
/// <summary> /// 此方法一般用来处理第一次同步通知失败的情况 /// </summary> public static void PushOrderStatusToCustomer() { //1.0 获取配置信息 //2.0 解析站点对不同站点不同解析 //http://www.chancheng.gov.cn/ccjy/060102/bmlist2.shtml //demo /html/body/div[1]/div[2]/div/div[2]/div[2] var confilist = ibll.SX_SiteConfig.where (s => true).ToList(); foreach (var site in confilist) { List <SX_News> list = new List <SX_News>(); switch (site.Type) { case "HuiZhouShanXiangedu": list = CrawlerUtils.HuiZhouShanXiangedu(site); break; case "HZLongMen": list = CrawlerUtils.HZLongMen(site); break; case "HZShiZhi": list = CrawlerUtils.HZShiZhi(site); break; case "HZHuiCheng": list = CrawlerUtils.HZHuiCheng(site); break; case "HZDaYaWan1": list = CrawlerUtils.HZDaYaWan1(site); break; case "HZDaYaWan2": list = CrawlerUtils.HZDaYaWan2(site); break; case "HZZhongKai": list = CrawlerUtils.HZZhongKai(site); break; case "HZHuiDong": list = CrawlerUtils.HZHuiDong(site); break; case "HZHuiYang": list = CrawlerUtils.HZHuiYang(site); break; case "HZBoLuo": list = CrawlerUtils.HZBoLuo(site); break; //case "FoShanChanChengedu": // //佛山禅城教育局 // list = CrawlerUtils.FoShanChanChengedu(site); // break; //case "SWChengQu": // list = CrawlerUtils.SWChengQu(site); // break; //case "FSShunDe": // list = CrawlerUtils.FSShunDe(site); // break; //case "FSNanHai": // list = CrawlerUtils.FSNanHai(site); // break; //case "FSGaoMing1": // list = CrawlerUtils.FSGaoMing1(site); // break; //case "FSGaoMing2": // list = CrawlerUtils.FSGaoMing2(site); // break; //case "FSSanShui": // list = CrawlerUtils.FSSanShui(site); // break; } if (list != null && list.Count > 0) { foreach (var item in list) { if (ibll.SX_News.where (a => a.SiteName == item.SiteName && a.Url == item.Url).Count() == 0) { ibll.SX_News.Add(item); } } ibll.SX_News.SaveChanges(); LogHelper.Info <WindowControl>("{0}:获取{1}条数据", site.SiteName, list.Count); } } }