/// <summary>
/// 获取类型页面下案例
/// </summary>
/// <param name="param"></param>
public void SpiderPage(object param)
{
ArrayList al = (ArrayList)param;
int maxPageCount = Convert.ToInt32(al[0]);
int maxPageLength = Convert.ToInt32(al[1]);
string hostUrl = Convert.ToString(al[2]);
string pageUrl = Convert.ToString(al[3]);
string pageListIndexUrl = Convert.ToString(al[4]);
int rate = Convert.ToInt32(al[5]);
int pageCheckRate = Convert.ToInt32(al[6]);
int count = Convert.ToInt32(al[7]);
string pageRemark = Convert.ToString(al[8]);
string 页面分页链接参数 = pageUrl.Replace("_2_1_0", "_2_{0}_0");
string 页面下一页链接 = pageUrl;
int 当前总页数 = (count - 1) / maxPageLength + 1;
int 当前页码 = 1;
//获取上一次爬取到的页码
if (IsPrevious)
{
Dat_KeyValueConfig keyValueConfig = KeyValueConfigManager.GetKeyValueConfig(CityId, WebsiteManager.常州房产网_ID, pageUrl);
if (keyValueConfig != null)
{
当前页码 = Convert.ToInt32(keyValueConfig.KeyValue);
}
}
while (!string.IsNullOrEmpty(页面下一页链接))
{
string nowPageList = 页面下一页链接;
if (!页面下一页链接.ToLower().Contains("http://"))
{
nowPageList = hostUrl + 页面下一页链接;
}
log.Debug(string.Format("常州房产网SpiderHouse()--获取根页面下一页链接{0},hostUrl:{1}, pageListUrl:{2}, cityName{3}", nowPageList, hostUrl, pageListIndexUrl, CityName));
SpiderHouseByPageListUrl(hostUrl, nowPageList, rate, pageCheckRate, out 页面下一页链接);
//存储当前页面到数据库(用于吸取历史案例)
if (IsPrevious)
{
KeyValueConfigManager.SetKeyValueConfig(CityId, WebsiteManager.常州房产网_ID, pageUrl, 当前页码.ToString());
}
当前页码++;
//如果当前页码还不到最后一页
if (当前页码 <= 当前总页数)
{
页面下一页链接 = string.Format(页面分页链接参数, 当前页码.ToString());
}
else
{
break;
}
if (isNowPageStop)
{
break;
}
}
log.Debug(string.Format("常州房产网SpiderHouse()--获取{0}页面下信息吸取完成,{1},hostUrl:{2}, pageListUrl:{3}, cityName{4}", pageRemark, 页面下一页链接, hostUrl, pageListIndexUrl, CityName));
}
public static Dat_KeyValueConfig Update(Dat_KeyValueConfig obj, DataClass _db = null)
{
DataClass db = new DataClass(_db);
if (obj != null)
{
db.DB.Dat_KeyValueConfig_Update(obj.KeyName, obj.KeyValue, obj.WebId, obj.CityId, obj.ID);
}
return(obj);
}
public static Dat_KeyValueConfig GetKeyValueConfig(int cityId, int webId, string key, DataClass _db = null)
{
DataClass db = new DataClass(_db);
string sql = @"select * from Dat_KeyValueConfig with(nolock) where KeyName='{0}' and CityId={1} and WebId={2} ";
sql = string.Format(sql, key, cityId, webId);
Dat_KeyValueConfig obj = db.DB.ExecuteQuery <Dat_KeyValueConfig>(sql).FirstOrDefault();
return(obj);
}
public static Dat_KeyValueConfig Insert(Dat_KeyValueConfig obj, DataClass _db = null)
{
DataClass db = new DataClass(_db);
if (obj != null)
{
long nowID = 0;
db.DB.Dat_KeyValueConfig_Insert(obj.KeyName, obj.KeyValue, obj.WebId, obj.CityId, out nowID);
obj.ID = nowID;
}
return(obj);
}
public static bool SetKeyValueConfig(int cityId, int webId, string key, string value, DataClass _db = null)
{
DataClass db = new DataClass(_db);
Dat_KeyValueConfig obj = GetKeyValueConfig(cityId, webId, key, db);
if (obj != null)
{
Update(cityId, webId, key, value, db);
}
else
{
Insert(cityId, webId, key, value, db);
}
return(true);
}