public async Task ConsumeAsync(AsyncMessageConsumer <byte[]> consumer,
CancellationToken cancellationToken)
{
if (consumer.Registered)
{
throw new ApplicationException("This consumer is already registered");
}
if (!DeclareQueue(consumer.Queue))
{
throw new ApplicationException("Declare queue failed");
}
var channel = _channelDict[consumer.Queue];
consumer.Register();
consumer.OnClosing += x => { CloseQueue(x.Queue); };
await Task.Factory.StartNew(async() =>
{
while (await channel.Reader.WaitToReadAsync(cancellationToken))
{
var bytes = await channel.Reader.ReadAsync(cancellationToken);
Task.Factory.StartNew(async() =>
{
await consumer.InvokeAsync(bytes);
}, cancellationToken)
.ConfigureAwait(false).GetAwaiter();
}
}, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Default);
}
private async Task RegisterConsumerAsync(CancellationToken stoppingToken)
{
var topic = string.Format(TopicNames.Spider, Id.ToUpper());
Logger.LogInformation($"{Id} register topic {topic}");
_consumer = new MessageQueue.AsyncMessageConsumer <byte[]>(topic);
_consumer.Received += async bytes =>
{
var message = await bytes.DeserializeAsync(stoppingToken);
if (message == null)
{
Logger.LogWarning("{Id} received empty message");
return;
}
if (message is Exit exit)
{
Logger.LogInformation($"{Id} receive exit message {JsonConvert.SerializeObject(exit)}");
if (exit.SpiderId == Id.ToUpper())
{
await ExitAsync();
}
}
else if (message is Response response)
{
// 1. 从请求队列中去除请求
var request = _requestedQueue.Dequeue(response.RequestHash);
if (request == null)
{
Logger.LogWarning($"{Id} dequeue {response.RequestHash} failed");
}
else
{
if (response.StatusCode == HttpStatusCode.OK)
{
if (_services.IsDistributed)
{
Logger.LogInformation($"{Id} download {request.RequestUri}, {request.Hash} success");
}
request.Agent = response.Agent;
await _services.StatisticsClient.IncreaseAgentSuccessAsync(response.Agent,
response.ElapsedMilliseconds);
await HandleResponseAsync(request, response, bytes);
}
else
{
await _services.StatisticsClient.IncreaseAgentFailureAsync(response.Agent,
response.ElapsedMilliseconds);
var exception = Encoding.UTF8.GetString(response.Content.Data);
Logger.LogError(
$"{Id} download {request.RequestUri}, {request.Hash} status code: {response.StatusCode} failed: {exception}");
// 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列
await AddRequestsAsync(request);
OnError?.Invoke(request, response);
}
}
}
else
{
Logger.LogError($"{Id} receive error message {JsonConvert.SerializeObject(message)}");
}
};
await _services.MessageQueue.ConsumeAsync(_consumer, stoppingToken);
}