private async Task RqTestDefaultWorkflow(IRequestQueue rq, long crawlId)
{
TestContext.WriteLine($"CrawlId = {crawlId}");
var sw = new Stopwatch();
sw.Start();
// Enqueue 3 Urls
await rq.EnqueueAsync(crawlId, new[] { "j1", "j2", "j3" });
//Dequeue and finish 1
await foreach (var d in rq.DequeueAsync(crawlId, 1, DateTimeOffset.UtcNow.AddSeconds(10)))
{
TestContext.WriteLine($"[{crawlId}] rocessing... {d}");
await Task.Delay(200);
await rq.DeleteAsync(crawlId, d);
}
// Dequeue 1 and fail to finish in time
await foreach (var d in rq.DequeueAsync(crawlId, 1, DateTimeOffset.UtcNow.AddSeconds(-1)))
{
TestContext.WriteLine($"[{crawlId}] Failing... {d}");
await Task.Delay(200);
}
// Enqueue 2 more
rq.Enqueue(crawlId, new[] { "j4", "j5" });
// Dequeue up to 10
var d1 = rq.Dequeue(crawlId, 10, DateTimeOffset.UtcNow.AddSeconds(10));
foreach (var d in d1)
{
TestContext.WriteLine($"[{crawlId}] Processing... {d}");
}
rq.Delete(crawlId, d1);
// that should hav been 3 from the queue and 1 from the failed jobs
Assert.AreEqual(4, d1.Count);
// Queue should be empty
var d2 = rq.Dequeue(crawlId, 10, DateTimeOffset.UtcNow.AddSeconds(10));
Assert.AreEqual(0, d2.Count, message: $"[{crawlId}] Unexpected: {string.Join(",", d2)}");
sw.Stop();
TestContext.WriteLine($"[{crawlId}] ElapsedMilliseconds: {sw.ElapsedMilliseconds}");
}
private async Task NextRequest()
{
if (!this.Runing)
{
return;
}
while (!NeedWait)
{
var result = NextRequestFromScheduler();
if (result == null)
{
continue;
}
await this.NextRequest();
}
while (_startRequests.Count > 0)
{
var r = _startRequests.Dequeue();
this._scheduler.Push(r);
}
if (this.SpiderIsIdle)
{
this.Stop();
Console.WriteLine("done");
}
}
public Request Pop()
{
if (_cacheQueue.Count == 0)
{
return(null);
}
return(_cacheQueue.Dequeue());
}
private void Loop()
{
try {
while (true)
{
var request = _requestQeueue.Dequeue();
if (request == null)
{
Logger.LogInfo("No more requests to send. Time to terminate thread.");
break;
}
try {
SendRequest(request);
} catch (Exception e) {
OnSendRequestError(request, e);
}
}
} catch (Exception e) {
Logger.LogError(e, "Exception in SendRequestsThread.");
throw;
}
}