Esempio n. 1
0
        private async Task RegisterAgentAsync(string topic, CancellationToken stoppingToken)
        {
            var consumer = new AsyncMessageConsumer <byte[]>(topic);

            consumer.Received += HandleMessageAsync;
            await _messageQueue.ConsumeAsync(consumer, stoppingToken);

            _consumers.Add(consumer);
        }
Esempio n. 2
0
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            try
            {
                _logger.LogInformation("Agent center service is starting");

                await _agentStore.EnsureDatabaseAndTableCreatedAsync();

                _consumer           = new AsyncMessageConsumer <byte[]>(TopicNames.AgentCenter);
                _consumer.Received += async bytes =>
                {
                    var message = await bytes.DeserializeAsync(stoppingToken);

                    if (message == null)
                    {
                        _logger.LogWarning("Received empty message");
                        return;
                    }

                    if (message is Register register)
                    {
                        if (_distributed)
                        {
                            _logger.LogInformation($"Register agent: {register.AgentId}, {register.AgentName}");
                        }

                        await _agentStore.RegisterAsync(new AgentInfo(register.AgentId, register.AgentName,
                                                                      register.ProcessorCount,
                                                                      register.TotalMemory));
                    }
                    else if (message is Heartbeat heartbeat)
                    {
                        if (_distributed)
                        {
                            _logger.LogInformation($"Receive heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}");
                        }

                        await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName,
                                                                            heartbeat.FreeMemory, heartbeat.CpuLoad));
                    }
                    else
                    {
                        var msg = Encoding.UTF8.GetString(JsonSerializer.SerializeToUtf8Bytes(message));
                        _logger.LogWarning($"Not supported message: {msg}");
                    }
                };
                await _messageQueue.ConsumeAsync(_consumer, stoppingToken);

                _logger.LogInformation("Agent center service started");
            }
            catch (Exception e)
            {
                _logger.LogCritical(e.ToString());
            }
        }
Esempio n. 3
0
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            try
            {
                _logger.LogInformation("Agent center service is starting");

                await _agentStore.EnsureDatabaseAndTableCreatedAsync();

                _consumer           = new AsyncMessageConsumer <byte[]>(Topics.AgentCenter);
                _consumer.Received += async bytes =>
                {
                    object message;
                    try
                    {
                        message = await bytes.DeserializeAsync(stoppingToken);

                        if (message == null)
                        {
                            return;
                        }
                    }
                    catch (Exception e)
                    {
                        _logger.LogError($"Deserialize message failed: {e}");
                        return;
                    }

                    switch (message)
                    {
                    case Messages.Agent.Register register:
                    {
                        if (_distributed)
                        {
                            _logger.LogInformation($"Register agent: {register.AgentId}, {register.AgentName}");
                        }

                        await _agentStore.RegisterAsync(new AgentInfo(register.AgentId, register.AgentName,
                                                                      register.ProcessorCount,
                                                                      register.Memory));

                        break;
                    }

                    case Messages.Agent.Heartbeat heartbeat:
                    {
                        if (_distributed)
                        {
                            _logger.LogInformation(
                                $"Receive heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}");
                        }

                        await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName,
                                                                            heartbeat.AvailableMemory, heartbeat.CpuLoad));

                        break;
                    }

                    default:
                    {
                        var msg = JsonSerializer.Serialize(message);
                        _logger.LogWarning($"Message not supported: {msg}");
                        break;
                    }
                    }
                };
                await _messageQueue.ConsumeAsync(_consumer, stoppingToken);

                _logger.LogInformation("Agent center service started");
            }
            catch (Exception e)
            {
                _logger.LogCritical(e.ToString());
            }
        }
Esempio n. 4
0
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            _logger.LogInformation("Statistics service starting");
            await _statisticsStore.EnsureDatabaseAndTableCreatedAsync();

            _consumer           = new AsyncMessageConsumer <byte[]>(Topics.Statistics);
            _consumer.Received += async bytes =>
            {
                var message = await bytes.DeserializeAsync(stoppingToken);

                if (message == null)
                {
                    _logger.LogWarning("Received empty message");
                    return;
                }

                if (message is Messages.Statistics.Success success)
                {
                    await _statisticsStore.IncreaseSuccessAsync(success.SpiderId);
                }
                else if (message is Messages.Statistics.Start start)
                {
                    await _statisticsStore.StartAsync(start.SpiderId, start.SpiderName);
                }
                else if (message is Messages.Statistics.Failure failure)
                {
                    await _statisticsStore.IncreaseFailureAsync(failure.SpiderId);
                }
                else if (message is Messages.Statistics.Total total)
                {
                    await _statisticsStore.IncreaseTotalAsync(total.SpiderId, total.Count);
                }
                else if (message is Messages.Statistics.Exit exit)
                {
                    await _statisticsStore.ExitAsync(exit.SpiderId);
                }
                else if (message is Messages.Statistics.RegisterAgent registerAgent)
                {
                    await _statisticsStore.RegisterAgentAsync(registerAgent.AgentId, registerAgent.AgentName);
                }
                else if (message is Messages.Statistics.AgentSuccess agentSuccess)
                {
                    await _statisticsStore.IncreaseAgentSuccessAsync(agentSuccess.AgentId,
                                                                     agentSuccess.ElapsedMilliseconds);
                }
                else if (message is Messages.Statistics.AgentFailure agentFailure)
                {
                    await _statisticsStore.IncreaseAgentFailureAsync(agentFailure.AgentId,
                                                                     agentFailure.ElapsedMilliseconds);
                }
                else if (message is Messages.Statistics.Print print)
                {
                    var statistics = await _statisticsStore.GetSpiderStatisticsAsync(print.SpiderId);

                    if (statistics != null)
                    {
                        var left = statistics.Total >= statistics.Success
                                                        ? (statistics.Total - statistics.Success - statistics.Failure).ToString()
                                                        : "unknown";
                        var now   = DateTimeOffset.Now;
                        var speed = (decimal)(statistics.Success /
                                              (now - (statistics.Start ?? now.AddMinutes(-1))).TotalSeconds);
                        _logger.LogInformation(
                            $"{print.SpiderId} total {statistics.Total}, speed: {decimal.Round(speed, 2)}, success {statistics.Success}, failure {statistics.Failure}, left {left}");
                    }
                }
                else
                {
                    var log = Encoding.UTF8.GetString(JsonSerializer.SerializeToUtf8Bytes(message));
                    _logger.LogWarning($"Not supported message: {log}");
                }
            };
            await _messageQueue.ConsumeAsync(_consumer, stoppingToken);

            _logger.LogInformation("Statistics service started");
        }