public static HTTPRequest HttpCallback(HTTPResponse response)
{
SpiderRequest request = response.request.Addition as SpiderRequest;
var sResponse = new SpiderResponse(request, response);
sResponse = RunResponseMiddleware(currentSpider._ResponseMiddleware, sResponse);
if (sResponse == null)
{
return(null);
}
else
{
var n = request.callback(sResponse);
if (!isRunning)
{
return(null);
}
if (n == null)
{
return(null);
}
n = RunRequestMiddleware(currentSpider._RequestMiddleware, n);
return(response.MakeRequest(n.method, n.Url, n, HttpCallback, n.Cookie, n.AdditionHeader, n.PostData));
}
}
static HTTPRequest RequestCallback(HTTPResponse response)
{
Log.log(count.ToString(), 1, "RequestCallback");
Log.log(response.body.Length.ToString(), 1, "RequestCallback");
//HtmlDocument doc = new HtmlDocument();
//doc.LoadHtml(response.body);
//Log.log(response.body, Log.INFO, "RequestCallback");
//var node = doc.DocumentNode.SelectNodes("//*[@id='IndustryNews']/ul[1]/li[8]/a");
//foreach (var item in node)
//{
// Log.log(item.InnerText, Log.INFO, "RequestCallback");
// File.WriteAllText("E://test.txt", item.InnerText);
//}
count++;
return(response.MakeRequest(HTTPMethod.GET, "http://www.oschina.net/", null, RequestCallback).ClearHeader());
}
