/// <summary>
/// 初始化大商所仓单数据
/// </summary>
public static void GetDSFDataRepository_First()
{
var ibll = OperationContext.BLLSession;
string[] years = new string[] { "2018" };
int[] months = new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 };
string[] days = new string[] { "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"
, "13", "14", "15", "16", "17", "18", "19", "20", "21", "22"
, "23", "24", "25", "26", "27", "28", "29", "30", "31" };
foreach (string year in years)
{
foreach (int month in months)
{
foreach (string day in days)
{
int d = Convert.ToInt32(string.Format("{0}{1}{2}", year, month + 1 < 10 ? "0" + (month + 1).ToString() : (month + 1).ToString(), day));
if (d < 20180616)
{
continue;
}
if (ibll.FDataReposInit.where (a => a.Date == d && a.TradeHouse == TradeHouseType.dce.ToString() && a.Type == InitContentType.Cangdan.ToString()).Count() < 1)
{
string url = string.Format("http://www.dce.com.cn/publicweb/quotesdata/wbillWeeklyQuotes.html?year={0}&month={1}&day={2}",
year, month, day);
var model = CrawlerUtils.GetDSFDataRepository_First(url);
ibll.FDataReposInit.Add(model);
int n = ibll.FDataReposInit.SaveChanges();
System.Console.WriteLine("大商：" + d);
}
}
}
}
}
public static void GetDSFDataRepository_Second()
{
var ibll = OperationContext.BLLSession;
int count = ibll.FDataReposInit.where (a => a.ID > 0 && a.TradeHouse == TradeHouseType.dce.ToString() && a.IsCheckFinish == 0).Count();
for (int page = 1; page <= (count / 10) + 1; page++)
{
List <FDataReposInit> list = ibll.FDataReposInit.where (a => a.ID > 0 && a.TradeHouse == TradeHouseType.dce.ToString() && a.IsCheckFinish == 0).OrderBy(s => s.ID).Take(10).ToList();
foreach (FDataReposInit model in list)
{
if (ibll.FDataRepository.where (a => a.Date == model.Date && a.TradeHouse == TradeHouseType.dce.ToString()).Count() < 1)
{
List <FDataRepository> resplist = CrawlerUtils.GetDSFDataRepository_Second(model.Content, model.Date);
foreach (var item in resplist)
{
ibll.FDataRepository.Add(item);
}
model.IsCheckFinish = 1;
ibll.FDataReposInit.Update(model, new string[] { "IsCheckFinish" });
int num = ibll.SaveChanges();
System.Console.WriteLine("大商Check：" + model.Date);
}
else
{
continue;
}
}
}
System.Console.WriteLine("大商Check：FINISH");
}
/// <summary>
/// 上海仓单初始化
/// 2011-20140516
/// </summary>
public static void GetSHFDataRepository_FirstD()
{
TaskStart:
try
{
var ibll = OperationContext.BLLSession;
int count = ibll.FDataReposInit.where (a => a.Date <= 20140516 && a.TradeHouse == TradeHouseType.shfe.ToString() && a.IsCheckFinish == 0).Count();
for (int page = 1; page <= (count / 10) + 1; page++)
{
List <FDataReposInit> list = ibll.FDataReposInit.where (a => a.Date <= 20140516 && a.TradeHouse == TradeHouseType.shfe.ToString() && a.IsCheckFinish == 0).OrderBy(s => s.ID).Take(10).ToList();
foreach (FDataReposInit model in list)
{
List <FDataRepository> resplist = CrawlerUtils.GetSHFDataRepository_FirstD(model.Content, model.Date);
foreach (var item in resplist)
{
ibll.FDataRepository.Add(item);
}
model.IsCheckFinish = 1;
ibll.FDataReposInit.Update(model, new string[] { "IsCheckFinish" });
int num = ibll.SaveChanges();
System.Console.WriteLine("上海2011-2014Check：" + model.Date);
}
}
System.Console.WriteLine("上海2011-2014Check：FINISH");
}
catch (Exception ex)
{
System.Console.WriteLine(string.Format("异常：{0} 开始休眠", ex.Message));
System.Threading.Thread.Sleep(60000);
goto TaskStart;
}
}
/// <summary>
/// 获取商品交易所仓单数据
/// 2011-
/// </summary>
public static void GetZZFDataRepository_First()
{
ZZTaskStart:
try
{
var ibll = OperationContext.BLLSession;
//string[] years = new string[] { "2011", "2012", "2013", "2014", "2015", "2016", "2017", "2018" };
string[] years = new string[] { "2018" };
int[] months = new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 };
string[] days = new string[] { "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"
, "13", "14", "15", "16", "17", "18", "19", "20", "21", "22"
, "23", "24", "25", "26", "27", "28", "29", "30", "31" };
foreach (string year in years)
{
foreach (int month in months)
{
foreach (string day in days)
{
int date = Convert.ToInt32(string.Format("{0}{1}{2}", year, month + 1 < 10 ? "0" + (month + 1).ToString() : (month + 1).ToString(), day));
if (date < 20180616)
{
continue;
}
if (ibll.FDataReposInit.where (a => a.Date == date && a.TradeHouse == TradeHouseType.czce.ToString() && a.Type == InitContentType.Cangdan.ToString()).Count() < 1)
{
string url = "";
if (date < 20151001)
{
url = string.Format("http://www.czce.com.cn/portal/exchange/{0}/datawhsheet/{1}.htm", year, date);
}
else if (date < 20180616)
{
url = string.Format("http://www.czce.com.cn/portal/DFSStaticFiles/Future/{0}/{1}/FutureDataWhsheet.htm", year, date);
}
else
{
url = string.Format("http://www.czce.com.cn/cn/DFSStaticFiles/Future/{0}/{1}/FutureDataWhsheet.htm", year, date);
}
var model = CrawlerUtils.GetZZFDataRepository_First(url, date);
if (model != null)
{
ibll.FDataReposInit.Add(model);
System.Console.WriteLine("郑州init：" + date);
}
}
}
int n = ibll.FDataReposInit.SaveChanges();
System.Console.WriteLine("郑州：Save --{0}", n);
}
}
System.Console.WriteLine("郑州：FINISH");
}
catch (Exception ex)
{
System.Console.WriteLine("郑商所仓单：{0}", ex.Message);
System.Threading.Thread.Sleep(60000);
goto ZZTaskStart;
}
}
/// <summary>
/// 20140519-2018 上海仓单数据
/// </summary>
public static void GetSHFDataRepository_Second()
{
var ibll = OperationContext.BLLSession;
string[] years = new string[] { "2018" };
int[] months = new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 };
string[] days = new string[] { "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"
, "13", "14", "15", "16", "17", "18", "19", "20", "21", "22"
, "23", "24", "25", "26", "27", "28", "29", "30", "31" };
foreach (string year in years)
{
foreach (int month in months)
{
foreach (string day in days)
{
int date = Convert.ToInt32(string.Format("{0}{1}{2}", year, month + 1 < 10 ? "0" + (month + 1).ToString() : (month + 1).ToString(), day));
if (date <= 20180616)
{
continue;
}
if (ibll.FDataReposInit.where (a => a.Date == date && a.TradeHouse == TradeHouseType.shfe.ToString() && a.Type == InitContentType.Cangdan.ToString()).Count() < 1)
{
string url = string.Format("http://www.shfe.com.cn/data/dailydata/{0}dailystock.dat",
date);
var model = CrawlerUtils.GetSHFDataRepository_Second(url, date.ToString());
if (model != null)
{
ibll.FDataReposInit.Add(model);
System.Console.WriteLine("上海：" + date);
}
}
}
int n = ibll.FDataReposInit.SaveChanges();
System.Console.WriteLine("上海：Save --{0}", n);
}
}
System.Console.WriteLine("上海：FINISH");
}
/// <summary>
///
/// </summary>
/// <param name="args"></param>
static void Main(string[] args)
{
//string str = "{\"VARNAME\":\"铜$$COPPER\",\"VARSORT\":0,\"REGNAME\":\"上海$$Shanghai\",\"REGSORT\":0,\"WHABBRNAME\":\"期晟公司$$Qisheng\",\"WHROWS\":12,\"WGHTUNIT\":\"2\",\"WRTWGHTS\":1919,\"WRTCHANGE\":0,\"ROWORDER\":1,\"ROWSTATUS\":\"0\"}";
//string s = Regex.Replace(str, "(,\"VARSORT\".+?,)", ",");
//s = Regex.Replace(s, "(,\"REGSORT\".+?,)", ",");
//s = Regex.Replace(s, "(,\"WHROWS\".+?,)", ",");
//System.Console.WriteLine(s);
//s = Regex.Replace(s, "(,\"ROWORDER\".+?,)", ",");
//s = Regex.Replace(s, "(,\"WGHTUNIT\".+?,)", ",");
//s = Regex.Replace(s, "(,\"ROWSTATUS\".+?})", "}");
//s = Regex.Replace(s, "(\\$\\$.+?\")", "\"");
//System.Console.WriteLine(s);
//string title = "品种：苹果AP单位：张日期：2018-06-19";
//string temp = Regex.Match(title, "(品种：.+单位：)").ToString();
//System.Console.WriteLine(temp);
//temp = temp.Replace("品种：", "").Replace("单位：", "");
//System.Console.WriteLine(temp);
//System.Console.WriteLine("---------------------------------------------------------------");
//System.Console.ReadKey();
//var ibll = OperationContext.BLLSession;
//string str = "/dalianshangpin/yw/fw/jystz/ywtz/6095044/index.html";
//string no = Regex.Match(str, "(/[0-9]+/)").ToString();
//System.Console.WriteLine(no);
#region NewsTest
//var list = new List<FNews> { };
//var pageIndex = 1;
//while (pageIndex > 0 && pageIndex <= 2)
//{
// /* 郑州所 这个站点做了处理
// * 1.登录会话做了限制，会被重定向
// * 2.详情被生成为PDF格式，需要解析
// * */
// var news = CrawlerUtils.GetNewsFromUrl_ZZ(1, pageIndex);
// foreach (var item in news)
// {
// //ibll.FNews.Add(item);
// System.Console.WriteLine($"{item.AddDate.Value.ToShortDateString()} {item.NewsTitle}~({(item.NewContent.Length > 0 ? "有采集到详情" : "无")})");
// }
// /*
// int n = ibll.FNews.SaveChanges();
// if (n > 0)
// {
// System.Console.WriteLine("新闻：save-{0}", n);
// }
// */
// if (news.Count == 0)
// {
// pageIndex = 0;
// break;
// }
// // 请求太快貌似会被4O4
// Thread.Sleep(1000);
// pageIndex++;
//}
//System.Console.WriteLine("新闻：Finish");
#endregion
//SX_SiteConfig site = new SX_SiteConfig() { };
//site.SiteHost = "http://www.chancheng.gov.cn";
//site.SiteName = "佛山市禅城区教育局";
//site.SiteUrl = "http://www.chancheng.gov.cn/ccjy/060102/bmlist2.shtml";
//site.SiteListXPath = "/html/body/div[1]/div[2]/div/div[2]/div[2]";
//site.SiteItemXPath = "div/div";
//var list=CrawlerUtils.FoShanChanChengedu(site);
SX_SiteConfig site = new SX_SiteConfig()
{
};
site.SiteHost = "http://www.longmen.gov.cn";
site.SiteName = "山香教育惠州分校";
site.SiteUrl = "http://www.shanweirc.com/more.jsp?sortid=1701&pagex=1";
site.SiteListXPath = "//*[@id=\"person-news\"]";
site.SiteItemXPath = "li";
var list = CrawlerUtils.SWChengQu(site);
System.Console.WriteLine(list.Count);
System.Console.ReadKey();
}
/// <summary>
/// 此方法一般用来处理第一次同步通知失败的情况
/// </summary>
public static void PushOrderStatusToCustomer()
{
//1.0 获取配置信息
//2.0 解析站点对不同站点不同解析
//http://www.chancheng.gov.cn/ccjy/060102/bmlist2.shtml
//demo /html/body/div[1]/div[2]/div/div[2]/div[2]
var confilist = ibll.SX_SiteConfig.where (s => true).ToList();
foreach (var site in confilist)
{
List <SX_News> list = new List <SX_News>();
switch (site.Type)
{
case "HuiZhouShanXiangedu":
list = CrawlerUtils.HuiZhouShanXiangedu(site);
break;
case "HZLongMen":
list = CrawlerUtils.HZLongMen(site);
break;
case "HZShiZhi":
list = CrawlerUtils.HZShiZhi(site);
break;
case "HZHuiCheng":
list = CrawlerUtils.HZHuiCheng(site);
break;
case "HZDaYaWan1":
list = CrawlerUtils.HZDaYaWan1(site);
break;
case "HZDaYaWan2":
list = CrawlerUtils.HZDaYaWan2(site);
break;
case "HZZhongKai":
list = CrawlerUtils.HZZhongKai(site);
break;
case "HZHuiDong":
list = CrawlerUtils.HZHuiDong(site);
break;
case "HZHuiYang":
list = CrawlerUtils.HZHuiYang(site);
break;
case "HZBoLuo":
list = CrawlerUtils.HZBoLuo(site);
break;
//case "FoShanChanChengedu":
// //佛山禅城教育局
// list = CrawlerUtils.FoShanChanChengedu(site);
// break;
//case "SWChengQu":
// list = CrawlerUtils.SWChengQu(site);
// break;
//case "FSShunDe":
// list = CrawlerUtils.FSShunDe(site);
// break;
//case "FSNanHai":
// list = CrawlerUtils.FSNanHai(site);
// break;
//case "FSGaoMing1":
// list = CrawlerUtils.FSGaoMing1(site);
// break;
//case "FSGaoMing2":
// list = CrawlerUtils.FSGaoMing2(site);
// break;
//case "FSSanShui":
// list = CrawlerUtils.FSSanShui(site);
// break;
}
if (list != null && list.Count > 0)
{
foreach (var item in list)
{
if (ibll.SX_News.where (a => a.SiteName == item.SiteName && a.Url == item.Url).Count() == 0)
{
ibll.SX_News.Add(item);
}
}
ibll.SX_News.SaveChanges();
LogHelper.Info <WindowControl>("{0}:获取{1}条数据", site.SiteName, list.Count);
}
}
}
