예제 #1
0
        private async Task RegisterAgentAsync(string topic, CancellationToken stoppingToken)
        {
            var consumer = new MessageQueue.AsyncMessageConsumer <byte[]>(topic);

            consumer.Received += HandleMessageAsync;
            await _messageQueue.ConsumeAsync(consumer, stoppingToken);

            _consumers.Add(consumer);
        }
예제 #2
0
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            _logger.LogInformation("Agent register service starting");

            await _agentStore.EnsureDatabaseAndTableCreatedAsync();

            _consumer           = new AsyncMessageConsumer <byte[]>(TopicNames.AgentCenter);
            _consumer.Received += async bytes =>
            {
                var message = await bytes.DeserializeAsync(stoppingToken);

                if (message == null)
                {
                    _logger.LogWarning("Received empty message");
                    return;
                }

                if (message is Register register)
                {
                    if (_distributed)
                    {
                        _logger.LogInformation($"Register agent: {register.AgentId}, {register.AgentName}");
                    }

                    await _agentStore.RegisterAsync(new AgentInfo(register.AgentId, register.AgentName, register.ProcessorCount,
                                                                  register.TotalMemory));
                }
                else if (message is Heartbeat heartbeat)
                {
                    if (_distributed)
                    {
                        _logger.LogInformation($"Heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}");
                    }

                    await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName,
                                                                        heartbeat.FreeMemory, heartbeat.CpuLoad));
                }
                else
                {
                    _logger.LogWarning($"Not supported message: {JsonConvert.SerializeObject(message)}");
                }
            };
            await _messageQueue.ConsumeAsync(_consumer, stoppingToken);

            _logger.LogInformation("Agent register service started");
        }
예제 #3
0
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            _logger.LogInformation("Statistics service starting");
            await _statisticsStore.EnsureDatabaseAndTableCreatedAsync();

            _consumer           = new MessageQueue.AsyncMessageConsumer <byte[]>(TopicNames.Statistics);
            _consumer.Received += async bytes =>
            {
                var message = await bytes.DeserializeAsync(stoppingToken);

                if (message == null)
                {
                    _logger.LogWarning("Received empty message");
                    return;
                }

                if (message is Success success)
                {
                    await _statisticsStore.IncreaseSuccessAsync(success.SpiderId);
                }
                else if (message is Start start)
                {
                    await _statisticsStore.StartAsync(start.SpiderId, start.SpiderName);
                }
                else if (message is Failure failure)
                {
                    await _statisticsStore.IncreaseFailureAsync(failure.SpiderId);
                }
                else if (message is Total total)
                {
                    await _statisticsStore.IncreaseTotalAsync(total.SpiderId, total.Count);
                }
                else if (message is Exit exit)
                {
                    await _statisticsStore.ExitAsync(exit.SpiderId);
                }
                else if (message is AgentSuccess agentSuccess)
                {
                    await _statisticsStore.IncreaseAgentSuccessAsync(agentSuccess.AgentId, agentSuccess.ElapsedMilliseconds);
                }
                else if (message is AgentFailure agentFailure)
                {
                    await _statisticsStore.IncreaseAgentFailureAsync(agentFailure.AgentId, agentFailure.ElapsedMilliseconds);
                }
                else if (message is Print print)
                {
                    var statistics = await _statisticsStore.GetSpiderStatisticsAsync(print.SpiderId);

                    if (statistics != null)
                    {
                        var left = statistics.Total >= statistics.Success
                            ? (statistics.Total - statistics.Success - statistics.Failure).ToString()
                            : "unknown";
                        _logger.LogInformation(
                            $"{print.SpiderId} total {statistics.Total}, success {statistics.Success}, failure {statistics.Failure}, left {left}");
                    }
                }
                else
                {
                    var log = Encoding.UTF8.GetString(JsonSerializer.SerializeToUtf8Bytes(message));
                    _logger.LogWarning($"Not supported message: {log}");
                }
            };
            await _messageQueue.ConsumeAsync(_consumer, stoppingToken);

            _logger.LogInformation("Statistics service started");
        }