/// <summary> /// 获取类型页面下案例 /// </summary> /// <param name="param"></param> public void SpiderPage(object param) { ArrayList al = (ArrayList)param; int maxPageCount = Convert.ToInt32(al[0]); int maxPageLength = Convert.ToInt32(al[1]); string hostUrl = Convert.ToString(al[2]); string pageUrl = Convert.ToString(al[3]); string pageListIndexUrl = Convert.ToString(al[4]); int rate = Convert.ToInt32(al[5]); int pageCheckRate = Convert.ToInt32(al[6]); int count = Convert.ToInt32(al[7]); string pageRemark = Convert.ToString(al[8]); string 页面分页链接参数 = pageUrl.Replace("_2_1_0", "_2_{0}_0"); string 页面下一页链接 = pageUrl; int 当前总页数 = (count - 1) / maxPageLength + 1; int 当前页码 = 1; //获取上一次爬取到的页码 if (IsPrevious) { Dat_KeyValueConfig keyValueConfig = KeyValueConfigManager.GetKeyValueConfig(CityId, WebsiteManager.常州房产网_ID, pageUrl); if (keyValueConfig != null) { 当前页码 = Convert.ToInt32(keyValueConfig.KeyValue); } } while (!string.IsNullOrEmpty(页面下一页链接)) { string nowPageList = 页面下一页链接; if (!页面下一页链接.ToLower().Contains("http://")) { nowPageList = hostUrl + 页面下一页链接; } log.Debug(string.Format("常州房产网SpiderHouse()--获取根页面下一页链接{0},hostUrl:{1}, pageListUrl:{2}, cityName{3}", nowPageList, hostUrl, pageListIndexUrl, CityName)); SpiderHouseByPageListUrl(hostUrl, nowPageList, rate, pageCheckRate, out 页面下一页链接); //存储当前页面到数据库(用于吸取历史案例) if (IsPrevious) { KeyValueConfigManager.SetKeyValueConfig(CityId, WebsiteManager.常州房产网_ID, pageUrl, 当前页码.ToString()); } 当前页码++; //如果当前页码还不到最后一页 if (当前页码 <= 当前总页数) { 页面下一页链接 = string.Format(页面分页链接参数, 当前页码.ToString()); } else { break; } if (isNowPageStop) { break; } } log.Debug(string.Format("常州房产网SpiderHouse()--获取{0}页面下信息吸取完成,{1},hostUrl:{2}, pageListUrl:{3}, cityName{4}", pageRemark, 页面下一页链接, hostUrl, pageListIndexUrl, CityName)); }
public static Dat_KeyValueConfig Update(Dat_KeyValueConfig obj, DataClass _db = null) { DataClass db = new DataClass(_db); if (obj != null) { db.DB.Dat_KeyValueConfig_Update(obj.KeyName, obj.KeyValue, obj.WebId, obj.CityId, obj.ID); } return(obj); }
public static Dat_KeyValueConfig GetKeyValueConfig(int cityId, int webId, string key, DataClass _db = null) { DataClass db = new DataClass(_db); string sql = @"select * from Dat_KeyValueConfig with(nolock) where KeyName='{0}' and CityId={1} and WebId={2} "; sql = string.Format(sql, key, cityId, webId); Dat_KeyValueConfig obj = db.DB.ExecuteQuery <Dat_KeyValueConfig>(sql).FirstOrDefault(); return(obj); }
public static Dat_KeyValueConfig Insert(Dat_KeyValueConfig obj, DataClass _db = null) { DataClass db = new DataClass(_db); if (obj != null) { long nowID = 0; db.DB.Dat_KeyValueConfig_Insert(obj.KeyName, obj.KeyValue, obj.WebId, obj.CityId, out nowID); obj.ID = nowID; } return(obj); }
public static bool SetKeyValueConfig(int cityId, int webId, string key, string value, DataClass _db = null) { DataClass db = new DataClass(_db); Dat_KeyValueConfig obj = GetKeyValueConfig(cityId, webId, key, db); if (obj != null) { Update(cityId, webId, key, value, db); } else { Insert(cityId, webId, key, value, db); } return(true); }