public static void ScapeDataByCountyOrZip()
{
LogHelper.log.Info("ScapeDataByCountyOrZip start");
var fileName = ConfigurationManager.AppSettings.Get("CountyZipFile").ToString();
var json = File.ReadAllText(fileName);
var list = JsonConvert.DeserializeObject <List <CountyZipModel> >(json);
DayCareScrape me = new DayCareScrape("http://www.dleg.state.mi.us");
foreach (var r in list)
{
if (r.UseCounty)
{
r.County = r.County.ToUpper();
var url = string.Format("/brs_cdc/rs_lfl.asp?cdc_name=&address=&cnty_name={0}&cdc_city=&cdc_zip=&ftype=DC&lic_name=&lic_nbr=&Search=Search&sorry=yes", r.County);
if (!me.ExtractDayByCounty(url))
{
r.UseCounty = false;
}
}
}
string nweJson = JsonConvert.SerializeObject(list);
System.IO.File.WriteAllText(fileName, nweJson);
LogHelper.log.Info("ScapeDataByCountyOrZip end");
}
public static List <DayCareModel> GetDetailData(List <ScrapeSource> list)
{
var resultList = new List <DayCareModel>();
DayCareScrape me = new DayCareScrape("http://www.dleg.state.mi.us");
foreach (var r in list)
{
try
{
Console.WriteLine(BuildLogMessage(r));
var result = me.ExtractDayCareDetailList("http://www.dleg.state.mi.us/brs_cdc/" + r.DetailUrl);
result.FacilityInformation.County = r.County;
if (string.IsNullOrEmpty(result.FacilityInformation.Name))
{
LogHelper.log.Info(BuildLogMessage(r) + " no data scraped");
LogHelper.log.Info(BuildLogMessage(r) + " scrape again");
try
{
result = me.ExtractDayCareDetailList("http://www.dleg.state.mi.us/brs_cdc/" + r.DetailUrl);
result.FacilityInformation.County = r.County;
resultList.Add(result);
}
catch (Exception subex)
{
LogHelper.log.Info(BuildLogMessage(r) + "exception:" + subex.ToString());
}
}
resultList.Add(result);
}
catch (Exception ex)
{
LogHelper.log.Info(BuildLogMessage(r) + "exception:" + ex.ToString());
if (ex.Message.IndexOf("The operation has timed out") >= 0 || ex.Message.IndexOf("无法连接到远程服务器") >= 0 || ex.Message.IndexOf("基础连接已经关闭") >= 0)
{
LogHelper.log.Info(BuildLogMessage(r) + " scrape again:");
try
{
var result = me.ExtractDayCareDetailList("http://www.dleg.state.mi.us/brs_cdc/" + r.DetailUrl);
result.FacilityInformation.County = r.County;
resultList.Add(result);
}
catch (Exception subex)
{
LogHelper.log.Info(BuildLogMessage(r) + "exception:" + subex.ToString());
}
}
}
Console.WriteLine(BuildLogMessage(r) + "- End...");
}
return(resultList);
}
public static List <ScrapeSource> GetAllDetailList()
{
LogHelper.log.Info("scrape data list start");
List <ScrapeSource> list = new List <ScrapeSource>();
var zipCodeList = new List <string>();
var file = ConfigurationManager.AppSettings.Get("CountyZipFile").ToString();
var json = File.ReadAllText(file);
var countyZipList = JsonConvert.DeserializeObject <List <CountyZipModel> >(json);
DayCareScrape me = new DayCareScrape("http://www.dleg.state.mi.us");
foreach (var c in countyZipList)
{
if (c.UseCounty)
{
Console.WriteLine("Scape by county:" + c.County);
try
{
var url = string.Format("/brs_cdc/rs_lfl.asp?cdc_name=&address=&cnty_name={0}&cdc_city=&cdc_zip=&ftype=DC&lic_name=&lic_nbr=&Search=Search&sorry=yes", c.County);
me.ExtractDayCareList(url, list, c.County, "");
}
catch (Exception ex)
{
LogHelper.log.Info("Scape by county:" + c.County + "exception:" + ex.ToString());
}
Console.WriteLine("Scape by county:" + c.County + "end...");
}
else
{
foreach (var r in c.ZipCodeList)
{
Console.WriteLine("Scape by zipCode:" + r.ZipCode);
try
{
var url = string.Format("/brs_cdc/rs_lfl.asp?cdc_name=&address=&cnty_name=%25&cdc_city=&cdc_zip={0}&ftype=DC&lic_name=&lic_nbr=&Search=Search&sorry=yes", r.ZipCode.ToString());
me.ExtractDayCareList(url, list, c.County, r.ZipCode);
}
catch (Exception ex)
{
LogHelper.log.Info("Scape by zipCode:" + r.ZipCode + "exception:" + ex.ToString());
}
Console.WriteLine("Scape by zipCode:" + r.ZipCode + "end...");
}
}
}
LogHelper.log.Info("scrape data list end");
return(list);
}
