public Task ConsumeAsync(AsyncMessageConsumer <byte[]> consumer, CancellationToken stoppingToken) { if (consumer.Registered) { throw new ApplicationException("This consumer is already registered"); } if (!_connection.IsConnected) { _connection.TryConnect(); } var channel = _connection.CreateModel(); var eventingBasicConsumer = new AsyncEventingBasicConsumer(channel); var queue = channel.QueueDeclare().QueueName; var topic = consumer.Queue; channel.QueueBind(queue: queue, _options.Exchange, routingKey: topic); eventingBasicConsumer.Received += async(model, ea) => { await consumer.InvokeAsync(ea.Body.ToArray()); channel.BasicAck(deliveryTag: ea.DeliveryTag, multiple: false); }; consumer.OnClosing += x => { channel.Close(); }; //7. 启动消费者 channel.BasicConsume(queue: queue, autoAck: false, consumer: eventingBasicConsumer); return(Task.CompletedTask); }
private async Task RegisterAgentAsync(string topic, CancellationToken stoppingToken) { var consumer = new AsyncMessageConsumer <byte[]>(topic); consumer.Received += HandleMessageAsync; await _messageQueue.ConsumeAsync(consumer, stoppingToken); _consumers.Add(consumer); }
public async Task Close() { var messageQueue = GetMessageQueue(); var queue = Guid.NewGuid().ToString("N"); var consumer = new AsyncMessageConsumer <byte[]>(queue); var counter = 0; consumer.Received += async bytes => { var message = (Message)await bytes.DeserializeAsync(default);
public async Task Consumer() { var messageQueue = GetMessageQueue(); var consumer = new AsyncMessageConsumer <byte[]>("test"); consumer.Received += bytes => null; await messageQueue.ConsumeAsync(consumer, default); messageQueue.CloseQueue("test"); }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { _logger.LogInformation("Agent center service is starting"); await _agentStore.EnsureDatabaseAndTableCreatedAsync(); _consumer = new AsyncMessageConsumer <byte[]>(TopicNames.AgentCenter); _consumer.Received += async bytes => { var message = await bytes.DeserializeAsync(stoppingToken); if (message == null) { _logger.LogWarning("Received empty message"); return; } if (message is Register register) { if (_distributed) { _logger.LogInformation($"Register agent: {register.AgentId}, {register.AgentName}"); } await _agentStore.RegisterAsync(new AgentInfo(register.AgentId, register.AgentName, register.ProcessorCount, register.TotalMemory)); } else if (message is Heartbeat heartbeat) { if (_distributed) { _logger.LogInformation($"Receive heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}"); } await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName, heartbeat.FreeMemory, heartbeat.CpuLoad)); } else { var msg = Encoding.UTF8.GetString(JsonSerializer.SerializeToUtf8Bytes(message)); _logger.LogWarning($"Not supported message: {msg}"); } }; await _messageQueue.ConsumeAsync(_consumer, stoppingToken); _logger.LogInformation("Agent center service started"); } catch (Exception e) { _logger.LogCritical(e.ToString()); } }
private async Task RegisterConsumerAsync(CancellationToken stoppingToken) { var topic = string.Format(TopicNames.Spider, Id.ToUpper()); _consumer = new AsyncMessageConsumer <byte[]>(topic); _consumer.Received += async(bytes) => { var message = await bytes.DeserializeAsync(stoppingToken); if (message == null) { Logger.LogWarning("Received empty message"); return; } if (message is Exit exit) { if (exit.Id == Id) { _services.ApplicationLifetime.StopApplication(); } } else if (message is Response response) { // 1. 从请求队列中去除请求 var request = _requestedQueue.Dequeue(response.RequestHash); if (response.StatusCode == HttpStatusCode.OK) { request.Agent = response.Agent; await _services.StatisticsClient.IncreaseAgentSuccessAsync(response.Agent, response.ElapsedMilliseconds); await HandleResponseAsync(request, response, bytes); } else { await _services.StatisticsClient.IncreaseAgentFailureAsync(response.Agent, response.ElapsedMilliseconds); var exception = Encoding.UTF8.GetString(response.Content.Data); if (_services.IsDistributed) { Logger.LogError($"Agent request {request.RequestUri} failed: {exception}"); } // 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列 await AddRequestsAsync(request); } } }; await _services.MessageQueue.ConsumeAsync(_consumer, stoppingToken); }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { _logger.LogInformation("Agent register service starting"); await _agentStore.EnsureDatabaseAndTableCreatedAsync(); _consumer = new AsyncMessageConsumer <byte[]>(TopicNames.AgentRegister); _consumer.Received += async bytes => { var message = await bytes.DeserializeAsync(stoppingToken); if (message == null) { _logger.LogWarning("Received empty message"); return; } if (message is Register register) { if (_distributed) { _logger.LogInformation($"Register agent: {register.Id}, {register.Name}"); } await _agentStore.RegisterAsync(new AgentInfo(register.Id, register.Name, register.ProcessorCount, register.TotalMemory)); } else if (message is Heartbeat heartbeat) { if (_distributed) { _logger.LogInformation($"Heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}"); } await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName, heartbeat.FreeMemory, heartbeat.CpuLoad)); } else { _logger.LogWarning($"Not supported message: {JsonConvert.SerializeObject(message)}"); } }; await _messageQueue.ConsumeAsync(_consumer, stoppingToken); _logger.LogInformation("Agent register service started"); }
public Task ConsumeAsync(AsyncMessageConsumer <byte[]> consumer, CancellationToken stoppingToken) { if (consumer.Registered) { throw new ApplicationException("This consumer is already registered"); } if (!_connection.IsConnected) { _connection.TryConnect(); } var channel = _connection.CreateModel(); var basicConsumer = new AsyncEventingBasicConsumer(channel); channel.QueueDeclare(consumer.Queue, true, false, true, null); channel.QueueBind(consumer.Queue, _options.Exchange, consumer.Queue); basicConsumer.Received += async(model, ea) => { try { await consumer.InvokeAsync(ea.Body.ToArray()); } finally { channel.BasicAck(ea.DeliveryTag, false); } }; consumer.OnClosing += x => { channel.Close(); }; //7. 启动消费者 channel.BasicConsume(consumer.Queue, false, basicConsumer); return(Task.CompletedTask); }
public Task DomainEventAsyncHandlers_CanBeInvoked() { return(AsyncMessageConsumer.Test( async c => { c.WithConfig <MessagingConfig>(); c.Build(); var domainEventSrv = c.Services.Resolve <IMessagingService>(); var evt = new MockDomainEvent(); await domainEventSrv.PublishAsync(evt); }, (IAppContainer c) => { var consumer = c.Services.Resolve <MockAsyncMessageConsumer>(); consumer.ExecutedHandlers.Should().HaveCount(3); consumer.ExecutedHandlers.Should().Contain("OnEvent1Async", "OnEvent2Async", "OnEvent3"); } )); }
public Task ConsumeAsync <TMessage>(AsyncMessageConsumer <TMessage> consumer, CancellationToken stoppingToken) { var topic = consumer.Queue; var channel = _modelDict.GetOrAdd(topic, CreateChannel); var consumer1 = new AsyncEventingBasicConsumer(channel); var queue = channel.QueueDeclare().QueueName; channel.QueueBind(queue: queue, _options.Exchange, routingKey: topic); consumer1.Received += async(model, ea) => { if (consumer is AsyncMessageConsumer <byte[]> bytesConsumer) { await bytesConsumer.InvokeAsync(ea.Body); } else { var message = (TMessage)await ea.Body.DeserializeAsync(stoppingToken); if (message != null) { await consumer.InvokeAsync(message); } } channel.BasicAck(deliveryTag: ea.DeliveryTag, multiple: false); }; consumer.OnClosing += () => { channel.Close(); }; //7. 启动消费者 channel.BasicConsume(queue: queue, autoAck: false, consumer: consumer1); return(Task.CompletedTask); }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { _logger.LogInformation("Agent center service is starting"); await _agentStore.EnsureDatabaseAndTableCreatedAsync(); _consumer = new AsyncMessageConsumer <byte[]>(Topics.AgentCenter); _consumer.Received += async bytes => { object message; try { message = await bytes.DeserializeAsync(stoppingToken); if (message == null) { return; } } catch (Exception e) { _logger.LogError($"Deserialize message failed: {e}"); return; } switch (message) { case Messages.Agent.Register register: { if (_distributed) { _logger.LogInformation($"Register agent: {register.AgentId}, {register.AgentName}"); } await _agentStore.RegisterAsync(new AgentInfo(register.AgentId, register.AgentName, register.ProcessorCount, register.Memory)); break; } case Messages.Agent.Heartbeat heartbeat: { if (_distributed) { _logger.LogInformation( $"Receive heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}"); } await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName, heartbeat.AvailableMemory, heartbeat.CpuLoad)); break; } default: { var msg = JsonSerializer.Serialize(message); _logger.LogWarning($"Message not supported: {msg}"); break; } } }; await _messageQueue.ConsumeAsync(_consumer, stoppingToken); _logger.LogInformation("Agent center service started"); } catch (Exception e) { _logger.LogCritical(e.ToString()); } }
private async Task RegisterConsumerAsync(CancellationToken stoppingToken) { var topic = string.Format(TopicNames.Spider, Id.ToUpper()); Logger.LogInformation($"{Id} register topic {topic}"); _consumer = new AsyncMessageConsumer <byte[]>(topic); _consumer.Received += async bytes => { var message = await bytes.DeserializeAsync(stoppingToken); if (message == null) { Logger.LogWarning("{Id} received empty message"); return; } if (message is Exit exit) { Logger.LogInformation($"{Id} receive exit message {JsonConvert.SerializeObject(exit)}"); if (exit.Id == Id.ToUpper()) { await ExitAsync(); } } else if (message is Response response) { // 1. 从请求队列中去除请求 var request = _requestedQueue.Dequeue(response.RequestHash); if (request == null) { Logger.LogWarning($"{Id} dequeue {response.RequestHash} failed"); } else { if (response.StatusCode == HttpStatusCode.OK) { if (_services.IsDistributed) { Logger.LogInformation($"{Id} download {request.RequestUri}, {request.Hash} success"); } request.Agent = response.Agent; await _services.StatisticsClient.IncreaseAgentSuccessAsync(response.Agent, response.ElapsedMilliseconds); await HandleResponseAsync(request, response, bytes); } else { await _services.StatisticsClient.IncreaseAgentFailureAsync(response.Agent, response.ElapsedMilliseconds); var exception = Encoding.UTF8.GetString(response.Content.Data); Logger.LogError( $"{Id} download {request.RequestUri}, {request.Hash} status code: {response.StatusCode} failed: {exception}"); // 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列 await AddRequestsAsync(request); OnError?.Invoke(request, response); } } } else { Logger.LogError($"{Id} receive error message {JsonConvert.SerializeObject(message)}"); } }; await _services.MessageQueue.ConsumeAsync(_consumer, stoppingToken); }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { _logger.LogInformation("Statistics service starting"); await _statisticsStore.EnsureDatabaseAndTableCreatedAsync(); _consumer = new AsyncMessageConsumer <byte[]>(TopicNames.Statistics); _consumer.Received += async bytes => { var message = await bytes.DeserializeAsync(stoppingToken); if (message == null) { _logger.LogWarning("Received empty message"); return; } if (message is Success success) { await _statisticsStore.IncreaseSuccessAsync(success.Id); } else if (message is Start start) { await _statisticsStore.StartAsync(start.Id, start.Name); } else if (message is Failure failure) { await _statisticsStore.IncreaseFailureAsync(failure.Id); } else if (message is Total total) { await _statisticsStore.IncreaseTotalAsync(total.Id, total.Count); } else if (message is Exit exit) { await _statisticsStore.ExitAsync(exit.Id); } else if (message is AgentSuccess agentSuccess) { await _statisticsStore.IncreaseAgentSuccessAsync(agentSuccess.Id, agentSuccess.ElapsedMilliseconds); } else if (message is AgentFailure agentFailure) { await _statisticsStore.IncreaseAgentFailureAsync(agentFailure.Id, agentFailure.ElapsedMilliseconds); } else if (message is Print print) { var statistics = await _statisticsStore.GetSpiderStatisticsAsync(print.Id); if (statistics != null) { var left = statistics.Total >= statistics.Success ? (statistics.Total - statistics.Success - statistics.Failure).ToString() : "unknown"; _logger.LogInformation( $"{print.Id} total {statistics.Total}, success {statistics.Success}, failure {statistics.Failure}, left {left}"); } } else { var log = Encoding.UTF8.GetString(JsonSerializer.SerializeToUtf8Bytes(message)); _logger.LogWarning($"Not supported message: {log}"); } }; await _messageQueue.ConsumeAsync(_consumer, stoppingToken); _logger.LogInformation("Statistics service started"); }
private async Task RegisterConsumerAsync(CancellationToken stoppingToken) { var topic = string.Format(Topics.Spider, SpiderId.Id); Logger.LogInformation($"{SpiderId} register topic {topic}"); _consumer = new AsyncMessageConsumer <byte[]>(topic); _consumer.Received += async bytes => { object message; try { message = await bytes.DeserializeAsync(stoppingToken); if (message == null) { return; } } catch (Exception e) { Logger.LogError($"Deserialize message failed: {e}"); return; } switch (message) { case Messages.Spider.Exit exit: { Logger.LogInformation( $"{SpiderId} receive exit message {System.Text.Json.JsonSerializer.Serialize(exit)}"); if (exit.SpiderId == SpiderId.Id) { await ExitAsync(); } break; } case Response response: { // 1. 从请求队列中去除请求 // 2. 若是 timeout 的请求,无法通过 Dequeue 获取,会通过 _requestedQueue.GetAllTimeoutList() 获取得到 var request = _requestedQueue.Dequeue(response.RequestHash); if (request != null) { if (response.StatusCode.IsSuccessStatusCode()) { request.Agent = response.Agent; if (IsDistributed) { Logger.LogInformation( $"{SpiderId} download {request.RequestUri}, {request.Hash} via {request.Agent} success"); } // 是否下载成功由爬虫来决定,则非 Agent 自身 await _services.StatisticsClient.IncreaseAgentSuccessAsync(response.Agent, response.ElapsedMilliseconds); await HandleResponseAsync(request, response, bytes); } else { await _services.StatisticsClient.IncreaseAgentFailureAsync(response.Agent, response.ElapsedMilliseconds); Logger.LogError( $"{SpiderId} download {request.RequestUri}, {request.Hash} status code: {response.StatusCode} failed: {response.ReasonPhrase}"); // 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列 await AddRequestsAsync(request); OnRequestError?.Invoke(request, response); } } break; } default: Logger.LogError( $"{SpiderId} receive error message {System.Text.Json.JsonSerializer.Serialize(message)}"); break; } }; await _services.MessageQueue.ConsumeAsync(_consumer, stoppingToken); }
private async Task RegisterAgentAsync(string topic, CancellationToken stoppingToken) { _consumers = new MessageQueue.AsyncMessageConsumer <byte[]>(topic); _consumers.Received += HandleMessageAsync; await _messageQueue.ConsumeAsync(_consumers, stoppingToken); }