public virtual void Start(string searchtext)
{
if (this.CheckIfStartNecessary())
{
ItemRequestParam param = new ItemRequestParam
{
Text = searchtext,
Keys = this.ConvertSearchString(searchtext)
};
int index = this.CurrentPageIndex;
this.IsComplated = false;
for (; this.CurrentPageIndex < TakePageCount + index; this.CurrentPageIndex++)
{
WebRequestHelper.RequestAsync(string.Format(this.SearchUrl, HttpUtility.UrlEncode(searchtext), this.CurrentPageIndex), this.CrawlerInfo.UserAgent, this.OnRequestSuccess, this.OnRequestError, param);
Interlocked.Increment(ref this._requestCounter);
}
}
}
public virtual bool CheckItemNeedCollect(ResultItem resultItem, ItemRequestParam param)
{
//启用搜索引擎筛选
return(true);
//启用软件筛选
if (resultItem != null &&
param != null &&
param.Keys != null &&
param.Keys.Length > 1)
{
string title = resultItem.Title.ToUpper();
foreach (var key in param.Keys.Skip(1))
{
if (!title.Contains(key.ToUpper()))
{
return(false);
}
}
}
return(true);
}
