public Task ConsumeAsync(AsyncMessageConsumer <byte[]> consumer,
CancellationToken stoppingToken)
{
if (consumer.Registered)
{
throw new ApplicationException("This consumer is already registered");
}
if (!_connection.IsConnected)
{
_connection.TryConnect();
}
var channel = _connection.CreateModel();
var eventingBasicConsumer = new AsyncEventingBasicConsumer(channel);
var queue = channel.QueueDeclare().QueueName;
var topic = consumer.Queue;
channel.QueueBind(queue: queue, _options.Exchange, routingKey: topic);
eventingBasicConsumer.Received += async(model, ea) =>
{
await consumer.InvokeAsync(ea.Body.ToArray());
channel.BasicAck(deliveryTag: ea.DeliveryTag, multiple: false);
};
consumer.OnClosing += x =>
{
channel.Close();
};
//7. 启动消费者
channel.BasicConsume(queue: queue, autoAck: false, consumer: eventingBasicConsumer);
return(Task.CompletedTask);
}
private async Task RegisterAgentAsync(string topic, CancellationToken stoppingToken)
{
var consumer = new AsyncMessageConsumer <byte[]>(topic);
consumer.Received += HandleMessageAsync;
await _messageQueue.ConsumeAsync(consumer, stoppingToken);
_consumers.Add(consumer);
}
public async Task Close()
{
var messageQueue = GetMessageQueue();
var queue = Guid.NewGuid().ToString("N");
var consumer = new AsyncMessageConsumer <byte[]>(queue);
var counter = 0;
consumer.Received += async bytes =>
{
var message = (Message)await bytes.DeserializeAsync(default);
public async Task Consumer()
{
var messageQueue = GetMessageQueue();
var consumer = new AsyncMessageConsumer <byte[]>("test");
consumer.Received += bytes => null;
await messageQueue.ConsumeAsync(consumer, default);
messageQueue.CloseQueue("test");
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
_logger.LogInformation("Agent center service is starting");
await _agentStore.EnsureDatabaseAndTableCreatedAsync();
_consumer = new AsyncMessageConsumer <byte[]>(TopicNames.AgentCenter);
_consumer.Received += async bytes =>
{
var message = await bytes.DeserializeAsync(stoppingToken);
if (message == null)
{
_logger.LogWarning("Received empty message");
return;
}
if (message is Register register)
{
if (_distributed)
{
_logger.LogInformation($"Register agent: {register.AgentId}, {register.AgentName}");
}
await _agentStore.RegisterAsync(new AgentInfo(register.AgentId, register.AgentName,
register.ProcessorCount,
register.TotalMemory));
}
else if (message is Heartbeat heartbeat)
{
if (_distributed)
{
_logger.LogInformation($"Receive heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}");
}
await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName,
heartbeat.FreeMemory, heartbeat.CpuLoad));
}
else
{
var msg = Encoding.UTF8.GetString(JsonSerializer.SerializeToUtf8Bytes(message));
_logger.LogWarning($"Not supported message: {msg}");
}
};
await _messageQueue.ConsumeAsync(_consumer, stoppingToken);
_logger.LogInformation("Agent center service started");
}
catch (Exception e)
{
_logger.LogCritical(e.ToString());
}
}
private async Task RegisterConsumerAsync(CancellationToken stoppingToken)
{
var topic = string.Format(TopicNames.Spider, Id.ToUpper());
_consumer = new AsyncMessageConsumer <byte[]>(topic);
_consumer.Received += async(bytes) =>
{
var message = await bytes.DeserializeAsync(stoppingToken);
if (message == null)
{
Logger.LogWarning("Received empty message");
return;
}
if (message is Exit exit)
{
if (exit.Id == Id)
{
_services.ApplicationLifetime.StopApplication();
}
}
else if (message is Response response)
{
// 1. 从请求队列中去除请求
var request = _requestedQueue.Dequeue(response.RequestHash);
if (response.StatusCode == HttpStatusCode.OK)
{
request.Agent = response.Agent;
await _services.StatisticsClient.IncreaseAgentSuccessAsync(response.Agent,
response.ElapsedMilliseconds);
await HandleResponseAsync(request, response, bytes);
}
else
{
await _services.StatisticsClient.IncreaseAgentFailureAsync(response.Agent,
response.ElapsedMilliseconds);
var exception = Encoding.UTF8.GetString(response.Content.Data);
if (_services.IsDistributed)
{
Logger.LogError($"Agent request {request.RequestUri} failed: {exception}");
}
// 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列
await AddRequestsAsync(request);
}
}
};
await _services.MessageQueue.ConsumeAsync(_consumer, stoppingToken);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Agent register service starting");
await _agentStore.EnsureDatabaseAndTableCreatedAsync();
_consumer = new AsyncMessageConsumer <byte[]>(TopicNames.AgentRegister);
_consumer.Received += async bytes =>
{
var message = await bytes.DeserializeAsync(stoppingToken);
if (message == null)
{
_logger.LogWarning("Received empty message");
return;
}
if (message is Register register)
{
if (_distributed)
{
_logger.LogInformation($"Register agent: {register.Id}, {register.Name}");
}
await _agentStore.RegisterAsync(new AgentInfo(register.Id, register.Name, register.ProcessorCount,
register.TotalMemory));
}
else if (message is Heartbeat heartbeat)
{
if (_distributed)
{
_logger.LogInformation($"Heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}");
}
await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName,
heartbeat.FreeMemory, heartbeat.CpuLoad));
}
else
{
_logger.LogWarning($"Not supported message: {JsonConvert.SerializeObject(message)}");
}
};
await _messageQueue.ConsumeAsync(_consumer, stoppingToken);
_logger.LogInformation("Agent register service started");
}
public Task ConsumeAsync(AsyncMessageConsumer <byte[]> consumer,
CancellationToken stoppingToken)
{
if (consumer.Registered)
{
throw new ApplicationException("This consumer is already registered");
}
if (!_connection.IsConnected)
{
_connection.TryConnect();
}
var channel = _connection.CreateModel();
var basicConsumer = new AsyncEventingBasicConsumer(channel);
channel.QueueDeclare(consumer.Queue,
true,
false,
true,
null);
channel.QueueBind(consumer.Queue, _options.Exchange, consumer.Queue);
basicConsumer.Received += async(model, ea) =>
{
try
{
await consumer.InvokeAsync(ea.Body.ToArray());
}
finally
{
channel.BasicAck(ea.DeliveryTag, false);
}
};
consumer.OnClosing += x =>
{
channel.Close();
};
//7. 启动消费者
channel.BasicConsume(consumer.Queue, false, basicConsumer);
return(Task.CompletedTask);
}
public Task DomainEventAsyncHandlers_CanBeInvoked()
{
return(AsyncMessageConsumer.Test(
async c =>
{
c.WithConfig <MessagingConfig>();
c.Build();
var domainEventSrv = c.Services.Resolve <IMessagingService>();
var evt = new MockDomainEvent();
await domainEventSrv.PublishAsync(evt);
},
(IAppContainer c) =>
{
var consumer = c.Services.Resolve <MockAsyncMessageConsumer>();
consumer.ExecutedHandlers.Should().HaveCount(3);
consumer.ExecutedHandlers.Should().Contain("OnEvent1Async", "OnEvent2Async", "OnEvent3");
}
));
}
public Task ConsumeAsync <TMessage>(AsyncMessageConsumer <TMessage> consumer,
CancellationToken stoppingToken)
{
var topic = consumer.Queue;
var channel = _modelDict.GetOrAdd(topic, CreateChannel);
var consumer1 = new AsyncEventingBasicConsumer(channel);
var queue = channel.QueueDeclare().QueueName;
channel.QueueBind(queue: queue, _options.Exchange, routingKey: topic);
consumer1.Received += async(model, ea) =>
{
if (consumer is AsyncMessageConsumer <byte[]> bytesConsumer)
{
await bytesConsumer.InvokeAsync(ea.Body);
}
else
{
var message = (TMessage)await ea.Body.DeserializeAsync(stoppingToken);
if (message != null)
{
await consumer.InvokeAsync(message);
}
}
channel.BasicAck(deliveryTag: ea.DeliveryTag, multiple: false);
};
consumer.OnClosing += () =>
{
channel.Close();
};
//7. 启动消费者
channel.BasicConsume(queue: queue, autoAck: false, consumer: consumer1);
return(Task.CompletedTask);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
_logger.LogInformation("Agent center service is starting");
await _agentStore.EnsureDatabaseAndTableCreatedAsync();
_consumer = new AsyncMessageConsumer <byte[]>(Topics.AgentCenter);
_consumer.Received += async bytes =>
{
object message;
try
{
message = await bytes.DeserializeAsync(stoppingToken);
if (message == null)
{
return;
}
}
catch (Exception e)
{
_logger.LogError($"Deserialize message failed: {e}");
return;
}
switch (message)
{
case Messages.Agent.Register register:
{
if (_distributed)
{
_logger.LogInformation($"Register agent: {register.AgentId}, {register.AgentName}");
}
await _agentStore.RegisterAsync(new AgentInfo(register.AgentId, register.AgentName,
register.ProcessorCount,
register.Memory));
break;
}
case Messages.Agent.Heartbeat heartbeat:
{
if (_distributed)
{
_logger.LogInformation(
$"Receive heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}");
}
await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName,
heartbeat.AvailableMemory, heartbeat.CpuLoad));
break;
}
default:
{
var msg = JsonSerializer.Serialize(message);
_logger.LogWarning($"Message not supported: {msg}");
break;
}
}
};
await _messageQueue.ConsumeAsync(_consumer, stoppingToken);
_logger.LogInformation("Agent center service started");
}
catch (Exception e)
{
_logger.LogCritical(e.ToString());
}
}
private async Task RegisterConsumerAsync(CancellationToken stoppingToken)
{
var topic = string.Format(TopicNames.Spider, Id.ToUpper());
Logger.LogInformation($"{Id} register topic {topic}");
_consumer = new AsyncMessageConsumer <byte[]>(topic);
_consumer.Received += async bytes =>
{
var message = await bytes.DeserializeAsync(stoppingToken);
if (message == null)
{
Logger.LogWarning("{Id} received empty message");
return;
}
if (message is Exit exit)
{
Logger.LogInformation($"{Id} receive exit message {JsonConvert.SerializeObject(exit)}");
if (exit.Id == Id.ToUpper())
{
await ExitAsync();
}
}
else if (message is Response response)
{
// 1. 从请求队列中去除请求
var request = _requestedQueue.Dequeue(response.RequestHash);
if (request == null)
{
Logger.LogWarning($"{Id} dequeue {response.RequestHash} failed");
}
else
{
if (response.StatusCode == HttpStatusCode.OK)
{
if (_services.IsDistributed)
{
Logger.LogInformation($"{Id} download {request.RequestUri}, {request.Hash} success");
}
request.Agent = response.Agent;
await _services.StatisticsClient.IncreaseAgentSuccessAsync(response.Agent,
response.ElapsedMilliseconds);
await HandleResponseAsync(request, response, bytes);
}
else
{
await _services.StatisticsClient.IncreaseAgentFailureAsync(response.Agent,
response.ElapsedMilliseconds);
var exception = Encoding.UTF8.GetString(response.Content.Data);
Logger.LogError(
$"{Id} download {request.RequestUri}, {request.Hash} status code: {response.StatusCode} failed: {exception}");
// 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列
await AddRequestsAsync(request);
OnError?.Invoke(request, response);
}
}
}
else
{
Logger.LogError($"{Id} receive error message {JsonConvert.SerializeObject(message)}");
}
};
await _services.MessageQueue.ConsumeAsync(_consumer, stoppingToken);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Statistics service starting");
await _statisticsStore.EnsureDatabaseAndTableCreatedAsync();
_consumer = new AsyncMessageConsumer <byte[]>(TopicNames.Statistics);
_consumer.Received += async bytes =>
{
var message = await bytes.DeserializeAsync(stoppingToken);
if (message == null)
{
_logger.LogWarning("Received empty message");
return;
}
if (message is Success success)
{
await _statisticsStore.IncreaseSuccessAsync(success.Id);
}
else if (message is Start start)
{
await _statisticsStore.StartAsync(start.Id, start.Name);
}
else if (message is Failure failure)
{
await _statisticsStore.IncreaseFailureAsync(failure.Id);
}
else if (message is Total total)
{
await _statisticsStore.IncreaseTotalAsync(total.Id, total.Count);
}
else if (message is Exit exit)
{
await _statisticsStore.ExitAsync(exit.Id);
}
else if (message is AgentSuccess agentSuccess)
{
await _statisticsStore.IncreaseAgentSuccessAsync(agentSuccess.Id, agentSuccess.ElapsedMilliseconds);
}
else if (message is AgentFailure agentFailure)
{
await _statisticsStore.IncreaseAgentFailureAsync(agentFailure.Id, agentFailure.ElapsedMilliseconds);
}
else if (message is Print print)
{
var statistics = await _statisticsStore.GetSpiderStatisticsAsync(print.Id);
if (statistics != null)
{
var left = statistics.Total >= statistics.Success
? (statistics.Total - statistics.Success - statistics.Failure).ToString()
: "unknown";
_logger.LogInformation(
$"{print.Id} total {statistics.Total}, success {statistics.Success}, failure {statistics.Failure}, left {left}");
}
}
else
{
var log = Encoding.UTF8.GetString(JsonSerializer.SerializeToUtf8Bytes(message));
_logger.LogWarning($"Not supported message: {log}");
}
};
await _messageQueue.ConsumeAsync(_consumer, stoppingToken);
_logger.LogInformation("Statistics service started");
}
private async Task RegisterConsumerAsync(CancellationToken stoppingToken)
{
var topic = string.Format(Topics.Spider, SpiderId.Id);
Logger.LogInformation($"{SpiderId} register topic {topic}");
_consumer = new AsyncMessageConsumer <byte[]>(topic);
_consumer.Received += async bytes =>
{
object message;
try
{
message = await bytes.DeserializeAsync(stoppingToken);
if (message == null)
{
return;
}
}
catch (Exception e)
{
Logger.LogError($"Deserialize message failed: {e}");
return;
}
switch (message)
{
case Messages.Spider.Exit exit:
{
Logger.LogInformation(
$"{SpiderId} receive exit message {System.Text.Json.JsonSerializer.Serialize(exit)}");
if (exit.SpiderId == SpiderId.Id)
{
await ExitAsync();
}
break;
}
case Response response:
{
// 1. 从请求队列中去除请求
// 2. 若是 timeout 的请求，无法通过 Dequeue 获取，会通过 _requestedQueue.GetAllTimeoutList() 获取得到
var request = _requestedQueue.Dequeue(response.RequestHash);
if (request != null)
{
if (response.StatusCode.IsSuccessStatusCode())
{
request.Agent = response.Agent;
if (IsDistributed)
{
Logger.LogInformation(
$"{SpiderId} download {request.RequestUri}, {request.Hash} via {request.Agent} success");
}
// 是否下载成功由爬虫来决定，则非 Agent 自身
await _services.StatisticsClient.IncreaseAgentSuccessAsync(response.Agent,
response.ElapsedMilliseconds);
await HandleResponseAsync(request, response, bytes);
}
else
{
await _services.StatisticsClient.IncreaseAgentFailureAsync(response.Agent,
response.ElapsedMilliseconds);
Logger.LogError(
$"{SpiderId} download {request.RequestUri}, {request.Hash} status code: {response.StatusCode} failed: {response.ReasonPhrase}");
// 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列
await AddRequestsAsync(request);
OnRequestError?.Invoke(request, response);
}
}
break;
}
default:
Logger.LogError(
$"{SpiderId} receive error message {System.Text.Json.JsonSerializer.Serialize(message)}");
break;
}
};
await _services.MessageQueue.ConsumeAsync(_consumer, stoppingToken);
}
private async Task RegisterAgentAsync(string topic, CancellationToken stoppingToken)
{
_consumers = new MessageQueue.AsyncMessageConsumer <byte[]>(topic);
_consumers.Received += HandleMessageAsync;
await _messageQueue.ConsumeAsync(_consumers, stoppingToken);
}
