public static HTTPRequest HttpCallback(HTTPResponse response)
{
SpiderRequest request = response.request.Addition as SpiderRequest;
var sResponse = new SpiderResponse(request, response);
sResponse = RunResponseMiddleware(currentSpider._ResponseMiddleware, sResponse);
if (sResponse == null)
{
return(null);
}
else
{
var n = request.callback(sResponse);
if (!isRunning)
{
return(null);
}
if (n == null)
{
return(null);
}
n = RunRequestMiddleware(currentSpider._RequestMiddleware, n);
return(response.MakeRequest(n.method, n.Url, n, HttpCallback, n.Cookie, n.AdditionHeader, n.PostData));
}
}
private static SpiderRequest GetNext()
{
SpiderRequest next = null;
while (isRunning)
{
requestList.TryDequeue(out next);
if (next != null)
{
next = RunRequestMiddleware(currentSpider._RequestMiddleware, next);
break;
}
else
{
if (isRunning == false)
{
next = null;
break;
}
lock (waitLock)
{
Monitor.Wait(waitLock);
}
}
}
return(next);
}
public static SpiderRequest Make(String Url, Spider.SpiderRequestCallback callback)
{
SpiderRequest newRequest = new SpiderRequest();
newRequest.Url = Url;
newRequest.callback = callback;
return(newRequest);
}
internal static void AddRequest(SpiderRequest request)
{
if (!isRunning)
{
return;
}
requestList.Enqueue(request);
lock (waitLock)
{
Monitor.Pulse(waitLock);
}
}
public static SpiderResponse Make(SpiderRequest request, HTTPResponse HttpResponse)
{
return(new SpiderResponse(request, HttpResponse));
}
public SpiderResponse(SpiderRequest request, HTTPResponse HttpResponse)
{
this.request = request;
this.httpResponse = HttpResponse;
}
public void AddRequest(SpiderRequest request)
{
Spider.AddRequest(request);
}
static SpiderRequest RunRequestMiddleware(List <RequestMiddlewareBase> middlewareList, SpiderRequest request)
{
foreach (var item in middlewareList)
{
request = item.Process(request);
}
return(request);
}
public static SpiderRequest MakeRequest(string Url, SpiderRequestCallback callback)
{
return(SpiderRequest.Make(Url, callback));
}