Example #1
0
        private async Task RqTestDefaultWorkflow(IRequestQueue rq, long crawlId)
        {
            TestContext.WriteLine($"CrawlId = {crawlId}");

            var sw = new Stopwatch();

            sw.Start();

            // Enqueue 3 Urls
            await rq.EnqueueAsync(crawlId, new[] { "j1", "j2", "j3" });

            //Dequeue and finish 1
            await foreach (var d in rq.DequeueAsync(crawlId, 1, DateTimeOffset.UtcNow.AddSeconds(10)))
            {
                TestContext.WriteLine($"[{crawlId}] rocessing... {d}");
                await Task.Delay(200);

                await rq.DeleteAsync(crawlId, d);
            }

            // Dequeue 1 and fail to finish in time
            await foreach (var d in rq.DequeueAsync(crawlId, 1, DateTimeOffset.UtcNow.AddSeconds(-1)))
            {
                TestContext.WriteLine($"[{crawlId}] Failing... {d}");
                await Task.Delay(200);
            }

            // Enqueue 2 more
            rq.Enqueue(crawlId, new[] { "j4", "j5" });

            // Dequeue up to 10
            var d1 = rq.Dequeue(crawlId, 10, DateTimeOffset.UtcNow.AddSeconds(10));

            foreach (var d in d1)
            {
                TestContext.WriteLine($"[{crawlId}] Processing... {d}");
            }
            rq.Delete(crawlId, d1);

            // that should hav been 3 from the queue and 1 from the failed jobs
            Assert.AreEqual(4, d1.Count);


            // Queue should be empty
            var d2 = rq.Dequeue(crawlId, 10, DateTimeOffset.UtcNow.AddSeconds(10));

            Assert.AreEqual(0, d2.Count, message: $"[{crawlId}] Unexpected: {string.Join(",", d2)}");

            sw.Stop();
            TestContext.WriteLine($"[{crawlId}] ElapsedMilliseconds: {sw.ElapsedMilliseconds}");
        }
Example #2
0
        private async Task NextRequest()
        {
            if (!this.Runing)
            {
                return;
            }

            while (!NeedWait)
            {
                var result = NextRequestFromScheduler();
                if (result == null)
                {
                    continue;
                }

                await this.NextRequest();
            }

            while (_startRequests.Count > 0)
            {
                var r = _startRequests.Dequeue();
                this._scheduler.Push(r);
            }

            if (this.SpiderIsIdle)
            {
                this.Stop();
                Console.WriteLine("done");
            }
        }
Example #3
0
 public Request Pop()
 {
     if (_cacheQueue.Count == 0)
     {
         return(null);
     }
     return(_cacheQueue.Dequeue());
 }
 private void Loop()
 {
     try {
         while (true)
         {
             var request = _requestQeueue.Dequeue();
             if (request == null)
             {
                 Logger.LogInfo("No more requests to send. Time to terminate thread.");
                 break;
             }
             try {
                 SendRequest(request);
             } catch (Exception e) {
                 OnSendRequestError(request, e);
             }
         }
     } catch (Exception e) {
         Logger.LogError(e, "Exception in SendRequestsThread.");
         throw;
     }
 }