private async Task RunAsync(CancellationToken stoppingToken)
{
var tuple = ComputeIntervalAndDequeueBatch(Options.Speed);
var sleepTimeLimit = Options.EmptySleepTime * 1000;
await Task.Factory.StartNew(async() =>
{
try
{
var pausedTime = 0;
var sleepTime = 0;
var printFlag = 0;
while (!stoppingToken.IsCancellationRequested)
{
printFlag += tuple.Interval;
if (printFlag >= 5000)
{
printFlag = 0;
// todo: distribute spider should not print
await _services.StatisticsClient.PrintAsync(Id);
}
if (_requestedQueue.Count > Options.RequestedQueueCount)
{
if (pausedTime > sleepTimeLimit)
{
Logger.LogInformation(
$"{Id} paused too much time");
break;
}
pausedTime += tuple.Interval;
Logger.LogInformation(
$"{Id} too much requests enqueued");
await Task.Delay(tuple.Interval, default);
continue;
}
pausedTime = 0;
var timeoutRequests = _requestedQueue.GetAllTimeoutList();
if (timeoutRequests.Length > 0)
{
foreach (var request in timeoutRequests)
{
Logger.LogWarning(
$"{Id} request {request.RequestUri}, {request.Hash} timeout");
}
await AddRequestsAsync(timeoutRequests);
RequestTimeout?.Invoke(timeoutRequests);
}
else
{
var requests = (await _services.Scheduler.DequeueAsync(tuple.Batch)).ToArray();
if (requests.Length > 0)
{
sleepTime = 0;
}
else
{
sleepTime += tuple.Interval;
if (sleepTime > sleepTimeLimit)
{
break;
}
}
foreach (var request in requests)
{
ConfigureRequest(request);
}
if (!await PublishRequestMessagesAsync(requests))
{
break;
}
}
await Task.Delay(tuple.Interval, default);
}
}
catch (Exception e)
{
Logger.LogError($"{Id} exited by exception: {e}");
}
finally
{
await ExitAsync();
}
}, stoppingToken);
}