protected async Task <List <SpiderProxyUriEntity> > GetProxyEntitiesAsync(SpiderHttpClient webClient, HttpMethod httpMethod, IList <string> urls, int estimatedCount = 0)
{
var entities = estimatedCount > 0 ? new List <SpiderProxyUriEntity>(estimatedCount) : new List <SpiderProxyUriEntity>();
int urlCount = urls.Count;
Task[] tasks = new Task[urlCount];
for (int i = 0; i < urlCount; i++)
{
tasks[i] = Task.Run(async() =>
{
var tempList = await GetProxyEntitiesAsync(webClient, httpMethod, urls[i]);
if (!tempList.IsNullOrEmpty())
{
lock (entities)
{
entities.AddRange(tempList);
}
}
});
await Task.Delay(RandomTool.NextIntSafely(4000, 6000));
}
await Task.WhenAll(tasks);
return(entities);
}
public override async Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam)
{
string encodedCityName = WebTool.UrlEncodeByW3C(searchParam.CityName);
string encodedKeyword = WebTool.UrlEncodeByW3C(searchParam.Keyword);
LaGouResponseDataCollection dataCollection = new LaGouResponseDataCollection();
using (var client = CreateHttpClient())
{
await TryInitCookiesAsync(client, encodedCityName, encodedKeyword, searchParam.SearchType);
await Task.Delay(100);
string sid = null;
using (var postContent = PcWebApiProvider.GetPositionAjaxContent(encodedKeyword, "1"))
{
string ajaxRsp = await PostPositionAjaxAsync(client, postContent, encodedCityName, encodedKeyword, searchParam.SearchType);
var data = PcWebApiProvider.CreateResponseData(ajaxRsp, out sid);
if (data != null)
{
dataCollection.AddResponseData(data);
}
}
for (int i = 2; i <= searchParam.MaxPage; i++)
{
await Task.Delay(RandomTool.NextIntSafely(4000, 6000));
using (var postContent = PcWebApiProvider.GetPositionAjaxContent(encodedKeyword, i.ToString(), sid))
{
string ajaxRsp = await PostPositionAjaxAsync(client, postContent, encodedCityName, encodedKeyword, searchParam.SearchType);
var data = PcWebApiProvider.CreateResponseData(ajaxRsp, out sid);
if (data != null)
{
dataCollection.AddResponseData(data);
}
}
}
}
dataCollection.FillPositions(searchParam.Keyword);
dataCollection.FillCompanies(searchParam.CityName);
return(dataCollection);
}