private async Task RqTestDefaultWorkflow(IRequestQueue rq, long crawlId)
{
TestContext.WriteLine($"CrawlId = {crawlId}");
var sw = new Stopwatch();
sw.Start();
// Enqueue 3 Urls
await rq.EnqueueAsync(crawlId, new[] { "j1", "j2", "j3" });
//Dequeue and finish 1
await foreach (var d in rq.DequeueAsync(crawlId, 1, DateTimeOffset.UtcNow.AddSeconds(10)))
{
TestContext.WriteLine($"[{crawlId}] rocessing... {d}");
await Task.Delay(200);
await rq.DeleteAsync(crawlId, d);
}
// Dequeue 1 and fail to finish in time
await foreach (var d in rq.DequeueAsync(crawlId, 1, DateTimeOffset.UtcNow.AddSeconds(-1)))
{
TestContext.WriteLine($"[{crawlId}] Failing... {d}");
await Task.Delay(200);
}
// Enqueue 2 more
rq.Enqueue(crawlId, new[] { "j4", "j5" });
// Dequeue up to 10
var d1 = rq.Dequeue(crawlId, 10, DateTimeOffset.UtcNow.AddSeconds(10));
foreach (var d in d1)
{
TestContext.WriteLine($"[{crawlId}] Processing... {d}");
}
rq.Delete(crawlId, d1);
// that should hav been 3 from the queue and 1 from the failed jobs
Assert.AreEqual(4, d1.Count);
// Queue should be empty
var d2 = rq.Dequeue(crawlId, 10, DateTimeOffset.UtcNow.AddSeconds(10));
Assert.AreEqual(0, d2.Count, message: $"[{crawlId}] Unexpected: {string.Join(",", d2)}");
sw.Stop();
TestContext.WriteLine($"[{crawlId}] ElapsedMilliseconds: {sw.ElapsedMilliseconds}");
}
