public Task ConsumeAsync(AsyncMessageConsumer <byte[]> consumer,
                                 CancellationToken stoppingToken)
        {
            if (consumer.Registered)
            {
                throw new ApplicationException("This consumer is already registered");
            }

            if (!_connection.IsConnected)
            {
                _connection.TryConnect();
            }

            var channel = _connection.CreateModel();
            var eventingBasicConsumer = new AsyncEventingBasicConsumer(channel);
            var queue = channel.QueueDeclare().QueueName;
            var topic = consumer.Queue;

            channel.QueueBind(queue: queue, _options.Exchange, routingKey: topic);
            eventingBasicConsumer.Received += async(model, ea) =>
            {
                await consumer.InvokeAsync(ea.Body.ToArray());

                channel.BasicAck(deliveryTag: ea.DeliveryTag, multiple: false);
            };
            consumer.OnClosing += x =>
            {
                channel.Close();
            };
            //7. 启动消费者
            channel.BasicConsume(queue: queue, autoAck: false, consumer: eventingBasicConsumer);

            return(Task.CompletedTask);
        }
Exemple #2
0
        private async Task RegisterAgentAsync(string topic, CancellationToken stoppingToken)
        {
            var consumer = new AsyncMessageConsumer <byte[]>(topic);

            consumer.Received += HandleMessageAsync;
            await _messageQueue.ConsumeAsync(consumer, stoppingToken);

            _consumers.Add(consumer);
        }
Exemple #3
0
        public async Task Close()
        {
            var messageQueue = GetMessageQueue();
            var queue        = Guid.NewGuid().ToString("N");
            var consumer     = new AsyncMessageConsumer <byte[]>(queue);
            var counter      = 0;

            consumer.Received += async bytes =>
            {
                var message = (Message)await bytes.DeserializeAsync(default);
Exemple #4
0
        public async Task Consumer()
        {
            var messageQueue = GetMessageQueue();
            var consumer     = new AsyncMessageConsumer <byte[]>("test");

            consumer.Received += bytes => null;
            await messageQueue.ConsumeAsync(consumer, default);

            messageQueue.CloseQueue("test");
        }
Exemple #5
0
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            try
            {
                _logger.LogInformation("Agent center service is starting");

                await _agentStore.EnsureDatabaseAndTableCreatedAsync();

                _consumer           = new AsyncMessageConsumer <byte[]>(TopicNames.AgentCenter);
                _consumer.Received += async bytes =>
                {
                    var message = await bytes.DeserializeAsync(stoppingToken);

                    if (message == null)
                    {
                        _logger.LogWarning("Received empty message");
                        return;
                    }

                    if (message is Register register)
                    {
                        if (_distributed)
                        {
                            _logger.LogInformation($"Register agent: {register.AgentId}, {register.AgentName}");
                        }

                        await _agentStore.RegisterAsync(new AgentInfo(register.AgentId, register.AgentName,
                                                                      register.ProcessorCount,
                                                                      register.TotalMemory));
                    }
                    else if (message is Heartbeat heartbeat)
                    {
                        if (_distributed)
                        {
                            _logger.LogInformation($"Receive heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}");
                        }

                        await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName,
                                                                            heartbeat.FreeMemory, heartbeat.CpuLoad));
                    }
                    else
                    {
                        var msg = Encoding.UTF8.GetString(JsonSerializer.SerializeToUtf8Bytes(message));
                        _logger.LogWarning($"Not supported message: {msg}");
                    }
                };
                await _messageQueue.ConsumeAsync(_consumer, stoppingToken);

                _logger.LogInformation("Agent center service started");
            }
            catch (Exception e)
            {
                _logger.LogCritical(e.ToString());
            }
        }
Exemple #6
0
        private async Task RegisterConsumerAsync(CancellationToken stoppingToken)
        {
            var topic = string.Format(TopicNames.Spider, Id.ToUpper());

            _consumer           = new AsyncMessageConsumer <byte[]>(topic);
            _consumer.Received += async(bytes) =>
            {
                var message = await bytes.DeserializeAsync(stoppingToken);

                if (message == null)
                {
                    Logger.LogWarning("Received empty message");
                    return;
                }

                if (message is Exit exit)
                {
                    if (exit.Id == Id)
                    {
                        _services.ApplicationLifetime.StopApplication();
                    }
                }
                else if (message is Response response)
                {
                    // 1. 从请求队列中去除请求
                    var request = _requestedQueue.Dequeue(response.RequestHash);
                    if (response.StatusCode == HttpStatusCode.OK)
                    {
                        request.Agent = response.Agent;
                        await _services.StatisticsClient.IncreaseAgentSuccessAsync(response.Agent,
                                                                                   response.ElapsedMilliseconds);
                        await HandleResponseAsync(request, response, bytes);
                    }
                    else
                    {
                        await _services.StatisticsClient.IncreaseAgentFailureAsync(response.Agent,
                                                                                   response.ElapsedMilliseconds);

                        var exception = Encoding.UTF8.GetString(response.Content.Data);
                        if (_services.IsDistributed)
                        {
                            Logger.LogError($"Agent request {request.RequestUri} failed: {exception}");
                        }

                        // 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列
                        await AddRequestsAsync(request);
                    }
                }
            };
            await _services.MessageQueue.ConsumeAsync(_consumer, stoppingToken);
        }
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            _logger.LogInformation("Agent register service starting");

            await _agentStore.EnsureDatabaseAndTableCreatedAsync();

            _consumer           = new AsyncMessageConsumer <byte[]>(TopicNames.AgentRegister);
            _consumer.Received += async bytes =>
            {
                var message = await bytes.DeserializeAsync(stoppingToken);

                if (message == null)
                {
                    _logger.LogWarning("Received empty message");
                    return;
                }

                if (message is Register register)
                {
                    if (_distributed)
                    {
                        _logger.LogInformation($"Register agent: {register.Id}, {register.Name}");
                    }

                    await _agentStore.RegisterAsync(new AgentInfo(register.Id, register.Name, register.ProcessorCount,
                                                                  register.TotalMemory));
                }
                else if (message is Heartbeat heartbeat)
                {
                    if (_distributed)
                    {
                        _logger.LogInformation($"Heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}");
                    }

                    await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName,
                                                                        heartbeat.FreeMemory, heartbeat.CpuLoad));
                }
                else
                {
                    _logger.LogWarning($"Not supported message: {JsonConvert.SerializeObject(message)}");
                }
            };
            await _messageQueue.ConsumeAsync(_consumer, stoppingToken);

            _logger.LogInformation("Agent register service started");
        }
Exemple #8
0
        public Task ConsumeAsync(AsyncMessageConsumer <byte[]> consumer,
                                 CancellationToken stoppingToken)
        {
            if (consumer.Registered)
            {
                throw new ApplicationException("This consumer is already registered");
            }

            if (!_connection.IsConnected)
            {
                _connection.TryConnect();
            }

            var channel       = _connection.CreateModel();
            var basicConsumer = new AsyncEventingBasicConsumer(channel);

            channel.QueueDeclare(consumer.Queue,
                                 true,
                                 false,
                                 true,
                                 null);
            channel.QueueBind(consumer.Queue, _options.Exchange, consumer.Queue);
            basicConsumer.Received += async(model, ea) =>
            {
                try
                {
                    await consumer.InvokeAsync(ea.Body.ToArray());
                }
                finally
                {
                    channel.BasicAck(ea.DeliveryTag, false);
                }
            };
            consumer.OnClosing += x =>
            {
                channel.Close();
            };
            //7. 启动消费者
            channel.BasicConsume(consumer.Queue, false, basicConsumer);

            return(Task.CompletedTask);
        }
Exemple #9
0
        public Task DomainEventAsyncHandlers_CanBeInvoked()
        {
            return(AsyncMessageConsumer.Test(
                       async c =>
            {
                c.WithConfig <MessagingConfig>();
                c.Build();

                var domainEventSrv = c.Services.Resolve <IMessagingService>();
                var evt = new MockDomainEvent();
                await domainEventSrv.PublishAsync(evt);
            },
                       (IAppContainer c) =>
            {
                var consumer = c.Services.Resolve <MockAsyncMessageConsumer>();
                consumer.ExecutedHandlers.Should().HaveCount(3);
                consumer.ExecutedHandlers.Should().Contain("OnEvent1Async", "OnEvent2Async", "OnEvent3");
            }
                       ));
        }
Exemple #10
0
        public Task ConsumeAsync <TMessage>(AsyncMessageConsumer <TMessage> consumer,
                                            CancellationToken stoppingToken)
        {
            var topic     = consumer.Queue;
            var channel   = _modelDict.GetOrAdd(topic, CreateChannel);
            var consumer1 = new AsyncEventingBasicConsumer(channel);
            var queue     = channel.QueueDeclare().QueueName;

            channel.QueueBind(queue: queue, _options.Exchange, routingKey: topic);
            consumer1.Received += async(model, ea) =>
            {
                if (consumer is AsyncMessageConsumer <byte[]> bytesConsumer)
                {
                    await bytesConsumer.InvokeAsync(ea.Body);
                }
                else
                {
                    var message = (TMessage)await ea.Body.DeserializeAsync(stoppingToken);

                    if (message != null)
                    {
                        await consumer.InvokeAsync(message);
                    }
                }

                channel.BasicAck(deliveryTag: ea.DeliveryTag, multiple: false);
            };

            consumer.OnClosing += () =>
            {
                channel.Close();
            };
            //7. 启动消费者
            channel.BasicConsume(queue: queue, autoAck: false, consumer: consumer1);

            return(Task.CompletedTask);
        }
Exemple #11
0
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            try
            {
                _logger.LogInformation("Agent center service is starting");

                await _agentStore.EnsureDatabaseAndTableCreatedAsync();

                _consumer           = new AsyncMessageConsumer <byte[]>(Topics.AgentCenter);
                _consumer.Received += async bytes =>
                {
                    object message;
                    try
                    {
                        message = await bytes.DeserializeAsync(stoppingToken);

                        if (message == null)
                        {
                            return;
                        }
                    }
                    catch (Exception e)
                    {
                        _logger.LogError($"Deserialize message failed: {e}");
                        return;
                    }

                    switch (message)
                    {
                    case Messages.Agent.Register register:
                    {
                        if (_distributed)
                        {
                            _logger.LogInformation($"Register agent: {register.AgentId}, {register.AgentName}");
                        }

                        await _agentStore.RegisterAsync(new AgentInfo(register.AgentId, register.AgentName,
                                                                      register.ProcessorCount,
                                                                      register.Memory));

                        break;
                    }

                    case Messages.Agent.Heartbeat heartbeat:
                    {
                        if (_distributed)
                        {
                            _logger.LogInformation(
                                $"Receive heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}");
                        }

                        await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName,
                                                                            heartbeat.AvailableMemory, heartbeat.CpuLoad));

                        break;
                    }

                    default:
                    {
                        var msg = JsonSerializer.Serialize(message);
                        _logger.LogWarning($"Message not supported: {msg}");
                        break;
                    }
                    }
                };
                await _messageQueue.ConsumeAsync(_consumer, stoppingToken);

                _logger.LogInformation("Agent center service started");
            }
            catch (Exception e)
            {
                _logger.LogCritical(e.ToString());
            }
        }
Exemple #12
0
        private async Task RegisterConsumerAsync(CancellationToken stoppingToken)
        {
            var topic = string.Format(TopicNames.Spider, Id.ToUpper());

            Logger.LogInformation($"{Id} register topic {topic}");
            _consumer           = new AsyncMessageConsumer <byte[]>(topic);
            _consumer.Received += async bytes =>
            {
                var message = await bytes.DeserializeAsync(stoppingToken);

                if (message == null)
                {
                    Logger.LogWarning("{Id} received empty message");
                    return;
                }

                if (message is Exit exit)
                {
                    Logger.LogInformation($"{Id} receive exit message {JsonConvert.SerializeObject(exit)}");
                    if (exit.Id == Id.ToUpper())
                    {
                        await ExitAsync();
                    }
                }
                else if (message is Response response)
                {
                    // 1. 从请求队列中去除请求
                    var request = _requestedQueue.Dequeue(response.RequestHash);
                    if (request == null)
                    {
                        Logger.LogWarning($"{Id} dequeue {response.RequestHash} failed");
                    }
                    else
                    {
                        if (response.StatusCode == HttpStatusCode.OK)
                        {
                            if (_services.IsDistributed)
                            {
                                Logger.LogInformation($"{Id} download {request.RequestUri}, {request.Hash} success");
                            }

                            request.Agent = response.Agent;
                            await _services.StatisticsClient.IncreaseAgentSuccessAsync(response.Agent,
                                                                                       response.ElapsedMilliseconds);
                            await HandleResponseAsync(request, response, bytes);
                        }
                        else
                        {
                            await _services.StatisticsClient.IncreaseAgentFailureAsync(response.Agent,
                                                                                       response.ElapsedMilliseconds);

                            var exception = Encoding.UTF8.GetString(response.Content.Data);
                            Logger.LogError(
                                $"{Id} download {request.RequestUri}, {request.Hash} status code: {response.StatusCode} failed: {exception}");
                            // 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列
                            await AddRequestsAsync(request);

                            OnError?.Invoke(request, response);
                        }
                    }
                }
                else
                {
                    Logger.LogError($"{Id} receive error message {JsonConvert.SerializeObject(message)}");
                }
            };
            await _services.MessageQueue.ConsumeAsync(_consumer, stoppingToken);
        }
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            _logger.LogInformation("Statistics service starting");
            await _statisticsStore.EnsureDatabaseAndTableCreatedAsync();

            _consumer           = new AsyncMessageConsumer <byte[]>(TopicNames.Statistics);
            _consumer.Received += async bytes =>
            {
                var message = await bytes.DeserializeAsync(stoppingToken);

                if (message == null)
                {
                    _logger.LogWarning("Received empty message");
                    return;
                }

                if (message is Success success)
                {
                    await _statisticsStore.IncreaseSuccessAsync(success.Id);
                }
                else if (message is Start start)
                {
                    await _statisticsStore.StartAsync(start.Id, start.Name);
                }
                else if (message is Failure failure)
                {
                    await _statisticsStore.IncreaseFailureAsync(failure.Id);
                }
                else if (message is Total total)
                {
                    await _statisticsStore.IncreaseTotalAsync(total.Id, total.Count);
                }
                else if (message is Exit exit)
                {
                    await _statisticsStore.ExitAsync(exit.Id);
                }
                else if (message is AgentSuccess agentSuccess)
                {
                    await _statisticsStore.IncreaseAgentSuccessAsync(agentSuccess.Id, agentSuccess.ElapsedMilliseconds);
                }
                else if (message is AgentFailure agentFailure)
                {
                    await _statisticsStore.IncreaseAgentFailureAsync(agentFailure.Id, agentFailure.ElapsedMilliseconds);
                }
                else if (message is Print print)
                {
                    var statistics = await _statisticsStore.GetSpiderStatisticsAsync(print.Id);

                    if (statistics != null)
                    {
                        var left = statistics.Total >= statistics.Success
                            ? (statistics.Total - statistics.Success - statistics.Failure).ToString()
                            : "unknown";
                        _logger.LogInformation(
                            $"{print.Id} total {statistics.Total}, success {statistics.Success}, failure {statistics.Failure}, left {left}");
                    }
                }
                else
                {
                    var log = Encoding.UTF8.GetString(JsonSerializer.SerializeToUtf8Bytes(message));
                    _logger.LogWarning($"Not supported message: {log}");
                }
            };
            await _messageQueue.ConsumeAsync(_consumer, stoppingToken);

            _logger.LogInformation("Statistics service started");
        }
Exemple #14
0
        private async Task RegisterConsumerAsync(CancellationToken stoppingToken)
        {
            var topic = string.Format(Topics.Spider, SpiderId.Id);

            Logger.LogInformation($"{SpiderId} register topic {topic}");
            _consumer           = new AsyncMessageConsumer <byte[]>(topic);
            _consumer.Received += async bytes =>
            {
                object message;
                try
                {
                    message = await bytes.DeserializeAsync(stoppingToken);

                    if (message == null)
                    {
                        return;
                    }
                }
                catch (Exception e)
                {
                    Logger.LogError($"Deserialize message failed: {e}");
                    return;
                }

                switch (message)
                {
                case Messages.Spider.Exit exit:
                {
                    Logger.LogInformation(
                        $"{SpiderId} receive exit message {System.Text.Json.JsonSerializer.Serialize(exit)}");
                    if (exit.SpiderId == SpiderId.Id)
                    {
                        await ExitAsync();
                    }

                    break;
                }

                case Response response:
                {
                    // 1. 从请求队列中去除请求
                    // 2. 若是 timeout 的请求,无法通过 Dequeue 获取,会通过 _requestedQueue.GetAllTimeoutList() 获取得到
                    var request = _requestedQueue.Dequeue(response.RequestHash);

                    if (request != null)
                    {
                        if (response.StatusCode.IsSuccessStatusCode())
                        {
                            request.Agent = response.Agent;

                            if (IsDistributed)
                            {
                                Logger.LogInformation(
                                    $"{SpiderId} download {request.RequestUri}, {request.Hash} via {request.Agent} success");
                            }

                            // 是否下载成功由爬虫来决定,则非 Agent 自身
                            await _services.StatisticsClient.IncreaseAgentSuccessAsync(response.Agent,
                                                                                       response.ElapsedMilliseconds);
                            await HandleResponseAsync(request, response, bytes);
                        }
                        else
                        {
                            await _services.StatisticsClient.IncreaseAgentFailureAsync(response.Agent,
                                                                                       response.ElapsedMilliseconds);

                            Logger.LogError(
                                $"{SpiderId} download {request.RequestUri}, {request.Hash} status code: {response.StatusCode} failed: {response.ReasonPhrase}");

                            // 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列
                            await AddRequestsAsync(request);

                            OnRequestError?.Invoke(request, response);
                        }
                    }

                    break;
                }

                default:
                    Logger.LogError(
                        $"{SpiderId} receive error message {System.Text.Json.JsonSerializer.Serialize(message)}");
                    break;
                }
            };

            await _services.MessageQueue.ConsumeAsync(_consumer, stoppingToken);
        }
Exemple #15
0
 private async Task RegisterAgentAsync(string topic, CancellationToken stoppingToken)
 {
     _consumers           = new MessageQueue.AsyncMessageConsumer <byte[]>(topic);
     _consumers.Received += HandleMessageAsync;
     await _messageQueue.ConsumeAsync(_consumers, stoppingToken);
 }