public static List <data> GetHistroyData(PublishMessage pmessage) { try { var dataList = new List <data>(); using (var seleminun = new PhantomJSDriver()) { string t = HttpUtility.UrlDecode(string.Format("{0}{1}", pmessage.DataTime.Value.Year, pmessage.DataTime.Value.Month.ToString("00"))); string url = string.Format("{0}&month={1}", pmessage.Url.Trim(), t); seleminun.Navigate().GoToUrl(url); Thread.Sleep(new Random().Next(1000, 2000)); var rowEs = seleminun.FindElements(By.XPath("//div[@class=\"row\"]/div/table/tbody/tr")); Console.WriteLine(rowEs.Count); string path = string.Format("{0}\\{1}", Environment.CurrentDirectory, "data"); { for (int rowIndex = 1; rowIndex < rowEs.Count; rowIndex++) { var trEs = rowEs[rowIndex].FindElements(By.XPath(".//td")); var tempdata = new data() { name = pmessage.Name }; tempdata.date = DateTime.Parse(trEs[0].Text); tempdata.aqi = trEs[1].Text; tempdata.rank = trEs[2].Text; int p25, p10, so2, co, no2, O3_8h; if (int.TryParse(trEs[3].Text, out p25)) { tempdata.pm2d5 = p25; } if (int.TryParse(trEs[4].Text, out p10)) { tempdata.pm10 = p10; } if (int.TryParse(trEs[5].Text, out so2)) { tempdata.so2 = so2; } if (int.TryParse(trEs[6].Text, out co)) { tempdata.co = co; } if (int.TryParse(trEs[7].Text, out no2)) { tempdata.no2 = no2; } if (int.TryParse(trEs[8].Text, out O3_8h)) { tempdata.no2 = no2; } tempdata.updatetime = DateTime.Now; dataList.Add(tempdata); } } } return(dataList); } catch (Exception err) { Console.WriteLine(err.Message); Console.WriteLine(err.StackTrace); return(null); } }
static void Main(string[] args) { string type = Console.ReadLine(); //factory.HostName = "10.16.21.19"; switch (type) { case "1": #region 发布消息 var factory = new RabbitMQ.Client.ConnectionFactory(); factory.UserName = "******"; factory.Password = "******"; factory.Port = 5672; factory.HostName = "47.101.160.52"; do { using (var connect = factory.CreateConnection()) { using (var model = connect.CreateModel()) { model.ExchangeDeclare("HistroyCity", "direct", true); var q = model.QueueDeclare("CitiesQueue", true, false, false, null); model.QueueBind("CitiesQueue", "HistroyCity", "CityPublished"); using (var cityTable = new HistoryAirDataEntities()) { int count = (int)(q.MessageCount); if (20 - count > 0) { var list = (from c in cityTable.publish where c.ispublished == false select c).Take(20 - count).ToArray(); if (list.Length == 0) { break; } foreach (var c in list) { var pm = new PublishMessage() { Name = c.name, Url = c.homeurl, PublishIndex = c.index, DataTime = c.datatime }; var message = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(pm)); Console.WriteLine(pm.Name); model.BasicPublish("HistroyCity", "CityPublished", null, message); var pC = cityTable.publish.First(r => r.index == c.index); pC.ispublished = true; } cityTable.SaveChanges(); } } } } Console.WriteLine("*****************"); Thread.Sleep(1200000); } while (true); #endregion break; case "2": #region 更新起始时间,无效的功能 var factory1 = new RabbitMQ.Client.ConnectionFactory(); factory1.UserName = "******"; factory1.Password = "******"; factory1.Port = 5672; factory1.HostName = "47.101.160.52"; using (var connect = factory1.CreateConnection()) { using (var model = connect.CreateModel()) { model.ExchangeDeclare("HistroyCity", "direct", true); var q = model.QueueDeclare("CitiesQueue", true, false, false, null); var consumer = new RabbitMQ.Client.Events.EventingBasicConsumer(model); model.BasicQos(0, 1, false); consumer.Received += (channel, ea) => { var body = ea.Body; var message = Encoding.UTF8.GetString(body); var p = JsonConvert.DeserializeObject <cities>(message); Thread.Sleep(new Random().Next(4000, 10000)); using (var entry = new HistoryAirDataEntities()) using (var seleminun = new ChromeDriver()) //using (var seleminun = new PhantomJSDriver()) { seleminun.Navigate().GoToUrl(p.homeurl); Thread.Sleep(new Random().Next(1000, 2000)); //获取起始时间 var times = seleminun.FindElements(By.XPath("//ul[@class=\"unstyled1\"]/li/a")); string timeT = null; if (times != null & times.Count > 0) { string[] t = times[0].GetAttribute("href").Split(new string[] { "=" }, StringSplitOptions.RemoveEmptyEntries); timeT = t.Last(); } //获取当前页码上的信息并写入记录 var rowEs = seleminun.FindElements(By.XPath("//div[@class=\"row\"]/div/table/tbody/tr")); Console.WriteLine(rowEs.Count); //string path = string.Format("{0}\\{1}", Environment.CurrentDirectory, "data"); //using (var sw = new StreamWriter(string.Format("{0}//{1}", path, p.name), true)) { for (int rowIndex = 1; rowIndex < rowEs.Count; rowIndex++) { var trEs = rowEs[rowIndex].FindElements(By.XPath(".//td")); var tempdata = new data() { name = p.name }; tempdata.date = DateTime.Parse(trEs[0].Text); tempdata.aqi = trEs[1].Text; tempdata.rank = trEs[2].Text; int off = 0; if (trEs.Count == 10) { off = 1; tempdata.range = trEs[3].Text; } int p25, p10, so2, no2, O3_8h; float co; if (int.TryParse(trEs[3 + off].Text, out p25)) { tempdata.pm2d5 = p25; } if (int.TryParse(trEs[4 + off].Text, out p10)) { tempdata.pm10 = p10; } if (int.TryParse(trEs[5 + off].Text, out so2)) { tempdata.so2 = so2; } if (float.TryParse(trEs[6 + off].Text, out co)) { tempdata.co = co; } if (int.TryParse(trEs[7 + off].Text, out no2)) { tempdata.no2 = no2; } if (int.TryParse(trEs[8 + off].Text, out O3_8h)) { tempdata.o3_8 = O3_8h; } tempdata.updatetime = DateTime.Now; entry.data.Add(tempdata); //var tempData = new Data() //{ // Date = trEs[0].Text, // AQI = trEs[1].Text, // Rank = trEs[2 + off].Text, // Pm2d5 = trEs[3 + off].Text, // Pm10 = trEs[4 + off].Text, // So2 = trEs[5 + off].Text, // Co = trEs[6 + off].Text, // No2 = trEs[7 + off].Text, // O3_8h = trEs[8 + off].Text //}; //if (trEs.Count == 10) //{ // tempData.Range = trEs[3].Text; //} //sw.WriteLine(JsonConvert.SerializeObject(tempData)); } var target = entry.cities.First(rr => rr.name == p.name); target.starttime = timeT; target.updatetime = DateTime.Now; Console.WriteLine("************"); Console.WriteLine(target.name); Console.WriteLine("************"); } entry.SaveChanges(); } Thread.Sleep(1000); model.BasicAck(ea.DeliveryTag, false); }; model.BasicConsume("CitiesQueue", false, consumer); var r = Console.ReadLine(); if (r.ToLower().Trim() == "over") { return; } } } #endregion break; case "3": #region 获取任务消息并进行数据获取 var factory2 = new RabbitMQ.Client.ConnectionFactory() { UserName = "******", Password = "******", Port = 5672, HostName = "47.101.160.52" }; using (var connect = factory2.CreateConnection()) { using (var model = connect.CreateModel()) { model.ExchangeDeclare("HistroyCity", "direct", true); var q = model.QueueDeclare("CitiesQueue", true, false, false, null); var consumer = new RabbitMQ.Client.Events.EventingBasicConsumer(model); model.BasicQos(0, 1, false); consumer.Received += (channel, ea) => { List <data> tempData = null; try { var body = ea.Body; var message = Encoding.UTF8.GetString(body); var p = JsonConvert.DeserializeObject <PublishMessage>(message); tempData = DataManager.GetHistroyData(p); if (tempData != null) { using (var enerty = new HistoryAirDataEntities()) using (var trans = enerty.Database.BeginTransaction()) { for (int i = 0; i < tempData.Count; i++) { enerty.data.Add(tempData[i]); } var reslut = enerty.SaveChanges(); if (reslut > 0) { trans.Commit(); } else { trans.Rollback(); } model.BasicAck(ea.DeliveryTag, false); } } else { model.BasicNack(ea.DeliveryTag, false, true); } } catch (Exception err) { Console.WriteLine(err.Message); Console.WriteLine(err.StackTrace); model.BasicNack(ea.DeliveryTag, false, true); } #region old //using (var entry = new HistoryAirDataEntities()) ////using (var seleminun = new ChromeDriver()) //using (var seleminun = new PhantomJSDriver()) //{ // Thread.Sleep(new Random().Next(240000, 600000)); // string t = HttpUtility.UrlDecode(string.Format("{0}{1}", p.DataTime.Value.Year, p.DataTime.Value.Month.ToString("00"))); // string url = string.Format("{0}&month={1}", p.Url.Trim(), t); // seleminun.Navigate().GoToUrl(url); // Thread.Sleep(new Random().Next(1000, 2000)); // var rowEs = seleminun.FindElements(By.XPath("//div[@class=\"row\"]/div/table/tbody/tr")); // Console.WriteLine(rowEs.Count); // string path = string.Format("{0}\\{1}", Environment.CurrentDirectory, "data"); // using (var sw = new StreamWriter(string.Format("{0}//{1}", path, p.Name), true)) // { // for (int rowIndex = 1; rowIndex < rowEs.Count; rowIndex++) // { // var trEs = rowEs[rowIndex].FindElements(By.XPath(".//td")); // var tempdata = new data() { name = p.Name }; // tempdata.date = DateTime.Parse(trEs[0].Text); // tempdata.aqi = trEs[1].Text; // tempdata.rank = trEs[2].Text; // int p25, p10, so2, co, no2, O3_8h; // if (int.TryParse(trEs[3].Text, out p25)) // { // tempdata.pm2d5 = p25; // } // if (int.TryParse(trEs[4].Text, out p10)) // { // tempdata.pm10 = p10; // } // if (int.TryParse(trEs[5].Text, out so2)) // { // tempdata.so2 = so2; // } // if (int.TryParse(trEs[6].Text, out co)) // { // tempdata.co = co; // } // if (int.TryParse(trEs[7].Text, out no2)) // { // tempdata.no2 = no2; // } // if (int.TryParse(trEs[8].Text, out O3_8h)) // { // tempdata.no2 = no2; // } // entry.data.Add(tempdata); // var tempData = new Data() // { // Date = trEs[0].Text, // AQI = trEs[1].Text, // Rank = trEs[2].Text, // Pm2d5 = trEs[3].Text, // Pm10 = trEs[4].Text, // So2 = trEs[5].Text, // Co = trEs[6].Text, // No2 = trEs[7].Text, // O3_8h = trEs[8].Text // }; // sw.WriteLine(JsonConvert.SerializeObject(tempData)); // } // } // entry.SaveChanges(); //} #endregion }; model.BasicConsume("CitiesQueue", false, consumer); var r = Console.ReadLine(); if (r.ToLower().Trim() == "over") { return; } } } #endregion break; case "4": #region 在数据库中插入任务计划 using (var cityTable = new HistoryAirDataEntities()) { var list = (from c in cityTable.cities select c).ToArray(); for (int yIndex = 2018; yIndex < 2019; yIndex++) { for (int mIndex = 11; mIndex < 12; mIndex++) { foreach (var c in list) { if (yIndex == 2013) { mIndex = 11; } else if (yIndex == DateTime.Now.Year) { if (mIndex + 1 >= DateTime.Now.Month) { continue; } } var p = new publish(); p.name = c.name; p.homeurl = c.homeurl; p.datatime = new DateTime(yIndex, mIndex + 1, 1); cityTable.publish.Add(p); } } } cityTable.SaveChanges(); Console.WriteLine(cityTable.publish.Count()); Console.WriteLine("*****************"); } Console.WriteLine("*****************"); #endregion break; default: break; } #region 获取城市列表并序列化为本地文件 //var selHome = new ChromeDriver(); //selHome.Navigate().GoToUrl("https://www.aqistudy.cn/historydata/"); //var liEs = selHome.FindElements(By.XPath("//ul[@class=\"unstyled\"]/div/li/a")); //List<HistoryCity> cities = new List<HistoryCity>(); //foreach (var item in liEs) //{ // var tempc = new HistoryCity(); // tempc.Name = item.Text.Trim(); // tempc.Url = item.GetAttribute("href"); // cities.Add(tempc); //} //selHome.Close(); //foreach (var item in cities) //{ // using (var sw = new StreamWriter(string.Format("{0}\\{1}", path,item.Name))) // { // sw.Write(JsonConvert.SerializeObject(item)); // } //} #endregion #region 将本地文件写入到数据库中 //List<HistoryCity> cities = new List<HistoryCity>(); //string path = string.Format("{0}\\{1}", Environment.CurrentDirectory, "HistroyCities"); //string[] files = Directory.GetFiles(path); //foreach (var item in files) //{ // using (var sr=new StreamReader(item)) // { // var tempStr = sr.ReadToEnd(); // HistoryCity tempCity = JsonConvert.DeserializeObject<HistoryCity>(tempStr); // cities.Add(tempCity); // } //} //if (!Directory.Exists(path)) //{ // Directory.CreateDirectory(path); //} //try //{ // using (var citiesEnty = new HistoryAirDataEntities()) // { // int count = 0; // foreach (var item in cities) // { // var tempcities = new cities(); // tempcities.name = item.Name; // tempcities.homeurl = item.Url; // citiesEnty.cities.Add(tempcities); // count++; // if (count >= 10) // { // // citiesEnty.SaveChanges(); // count = 0; // } // } // citiesEnty.SaveChanges(); // } //} //catch (Exception err) //{ // Console.Write(err.Message); //} #endregion #region 获取城市历史数据 //string path = Environment.CurrentDirectory + "//Data"; //if (!Directory.Exists(path)) //{ // Directory.CreateDirectory(path); //} //using (var seleminun = new ChromeDriver()) //{ // seleminun.Navigate().GoToUrl("https://www.aqistudy.cn/historydata/daydata.php?city=%E5%AE%89%E9%98%B3&month=201703"); // Thread.Sleep(new Random().Next(1000, 2000)); // var rowEs = seleminun.FindElements(By.XPath("//div[@class=\"row\"]/div/table/tbody/tr")); // Console.WriteLine(rowEs.Count); // List<Data> datalist = new List<Data>(); // for (int rowIndex = 1; rowIndex < rowEs.Count; rowIndex++) // { // var trEs = rowEs[rowIndex].FindElements(By.XPath(".//td")); // var tempData = new Data() // { // Date =DateTime.Parse(trEs[0].Text), // AQI = trEs[1].Text, // Rank = trEs[2].Text, // Pm2d5 = trEs[3].Text, // Pm10 = trEs[4].Text, // So2 = trEs[5].Text, // Co = trEs[6].Text, // No2 = trEs[7].Text, // O3_8h = trEs[8].Text // }; // datalist.Add(tempData); // } // using (var sw=new StreamWriter(string.Format("{0}/{1}",path,"datalist"))) // { // sw.Write(JsonConvert.SerializeObject(datalist)); // } // //var tds = rowEs[rowEs.Count - 1].FindElements(By.XPath(".//td")); // //for (int i = 0; i < tds.Count; i++) // //{ // // Console.WriteLine(tds[i].Text); // //} // Console.ReadLine(); //} #endregion #region 更新起始时间(未完成) //List<HistoryCity> cities = new List<HistoryCity>(); //string path = string.Format("{0}\\{1}", Environment.CurrentDirectory, "HistroyCities"); //string[] files = Directory.GetFiles(path); //foreach (var item in files) //{ // using (var sr = new StreamReader(item)) // { // var tempStr = sr.ReadToEnd(); // HistoryCity tempCity = JsonConvert.DeserializeObject<HistoryCity>(tempStr); // cities.Add(tempCity); // } //} //using (var cityTable=new HistoryAirDataEntities()) //{ // for (int i = 0; i < cities.Count; i++) // { // var city = (from c in cityTable.cities where c.name == cities[i].Name & c.startTime==null select c).ToArray(); // if (city.Length==1) // { // using (var selenum=new ChromeDriver()) // { // selenum.Navigate().GoToUrl(city[0].homeurl); // if (true) // { // } // } // city[0].startTime = null; // } // } //} #endregion }