Ejemplo n.º 1
0
        /// <summary>
        /// 启动统计服务中心
        /// </summary>
        /// <param name="cancellationToken"></param>
        /// <returns></returns>
        /// <exception cref="SpiderException"></exception>
        public Task StartAsync(CancellationToken cancellationToken)
        {
            if (_isRunning)
            {
                throw new SpiderException("统计中心正在运行中");
            }

            _logger.LogInformation("统计中心启动");

            _mq.Subscribe(Framework.StatisticsServiceTopic, async message =>
            {
                var commandMessage = message.ToCommandMessage();
                if (commandMessage == null)
                {
                    _logger.LogWarning($"接收到非法消息: {message}");
                    return;
                }

                switch (commandMessage.Command)
                {
                case "Success":
                    {
                        var ownerId = commandMessage.Message;
                        await _statisticsStore.IncrementSuccessAsync(ownerId);
                        break;
                    }

                case "Failed":
                    {
                        var data = commandMessage.Message.Split(',');
                        await _statisticsStore.IncrementFailedAsync(data[0], int.Parse(data[1]));
                        break;
                    }

                case "Start":
                    {
                        var ownerId = commandMessage.Message;
                        await _statisticsStore.StartAsync(ownerId);
                        break;
                    }

                case "Exit":
                    {
                        var ownerId = commandMessage.Message;
                        await _statisticsStore.ExitAsync(ownerId);
                        break;
                    }

                case "Total":
                    {
                        var data = commandMessage.Message.Split(',');
                        await _statisticsStore.IncrementTotalAsync(data[0], int.Parse(data[1]));

                        break;
                    }

                case "DownloadSuccess":
                    {
                        var data = commandMessage.Message.Split(',');
                        await _statisticsStore.IncrementDownloadSuccessAsync(data[0], int.Parse(data[1]),
                                                                             long.Parse(data[2]));
                        break;
                    }

                case "DownloadFailed":
                    {
                        var data = commandMessage.Message.Split(',');
                        await _statisticsStore.IncrementDownloadFailedAsync(data[0], int.Parse(data[1]),
                                                                            long.Parse(data[2]));
                        break;
                    }

                case "Print":
                    {
                        var ownerId    = commandMessage.Message;
                        var statistics = await _statisticsStore.GetSpiderStatisticsAsync(ownerId);
                        _logger.LogInformation(
                            $"任务 {ownerId} 总计 {statistics.Total}, 成功 {statistics.Success}, 失败 {statistics.Failed}, 剩余 {statistics.Total - statistics.Success - statistics.Failed}");
                        break;
                    }
                }
            });
#if NETFRAMEWORK
            return(DotnetSpider.Core.Framework.CompletedTask);
#else
            return(Task.CompletedTask);
#endif
        }
Ejemplo n.º 2
0
        private async Task HandleStatisticsMessageAsync(Event message)
        {
            if (string.IsNullOrWhiteSpace(message.Data))
            {
                _logger.LogWarning($"接收到空消息");
                return;
            }

            switch (message.Type)
            {
            case "Success":
            {
                var ownerId = message.Data;
                await _statisticsStore.IncrementSuccessAsync(ownerId);

                break;
            }

            case "Failed":
            {
                var data = message.Data.Split(',');
                await _statisticsStore.IncrementFailedAsync(data[0], int.Parse(data[1]));

                break;
            }

            case "Start":
            {
                var ownerId = message.Data;
                await _statisticsStore.StartAsync(ownerId);

                break;
            }

            case "Exit":
            {
                var ownerId = message.Data;
                await _statisticsStore.ExitAsync(ownerId);

                break;
            }

            case "Total":
            {
                var data = message.Data.Split(',');
                await _statisticsStore.IncrementTotalAsync(data[0], int.Parse(data[1]));

                break;
            }

            case "DownloadSuccess":
            {
                var data = message.Data.Split(',');
                await _statisticsStore.IncrementDownloadSuccessAsync(data[0], int.Parse(data[1]),
                                                                     long.Parse(data[2]));

                break;
            }

            case "DownloadFailed":
            {
                var data = message.Data.Split(',');
                await _statisticsStore.IncrementDownloadFailedAsync(data[0], int.Parse(data[1]),
                                                                    long.Parse(data[2]));

                break;
            }

            case "Print":
            {
                var ownerId    = message.Data;
                var statistics = await _statisticsStore.GetSpiderStatisticsAsync(ownerId);

                if (statistics != null)
                {
                    _logger.LogInformation(
                        $"任务 {ownerId} 总计 {statistics.Total}, 成功 {statistics.Success}, 失败 {statistics.Failed}, 剩余 {statistics.Total - statistics.Success - statistics.Failed}");
                }

                break;
            }
            }
        }
Ejemplo n.º 3
0
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            _logger.LogInformation("Statistics service starting");
            await _statisticsStore.EnsureDatabaseAndTableCreatedAsync();

            _consumer           = new MessageQueue.AsyncMessageConsumer <byte[]>(TopicNames.Statistics);
            _consumer.Received += async bytes =>
            {
                var message = await bytes.DeserializeAsync(stoppingToken);

                if (message == null)
                {
                    _logger.LogWarning("Received empty message");
                    return;
                }

                if (message is Success success)
                {
                    await _statisticsStore.IncreaseSuccessAsync(success.SpiderId);
                }
                else if (message is Start start)
                {
                    await _statisticsStore.StartAsync(start.SpiderId, start.SpiderName);
                }
                else if (message is Failure failure)
                {
                    await _statisticsStore.IncreaseFailureAsync(failure.SpiderId);
                }
                else if (message is Total total)
                {
                    await _statisticsStore.IncreaseTotalAsync(total.SpiderId, total.Count);
                }
                else if (message is Exit exit)
                {
                    await _statisticsStore.ExitAsync(exit.SpiderId);
                }
                else if (message is AgentSuccess agentSuccess)
                {
                    await _statisticsStore.IncreaseAgentSuccessAsync(agentSuccess.AgentId, agentSuccess.ElapsedMilliseconds);
                }
                else if (message is AgentFailure agentFailure)
                {
                    await _statisticsStore.IncreaseAgentFailureAsync(agentFailure.AgentId, agentFailure.ElapsedMilliseconds);
                }
                else if (message is Print print)
                {
                    var statistics = await _statisticsStore.GetSpiderStatisticsAsync(print.SpiderId);

                    if (statistics != null)
                    {
                        var left = statistics.Total >= statistics.Success
                            ? (statistics.Total - statistics.Success - statistics.Failure).ToString()
                            : "unknown";
                        _logger.LogInformation(
                            $"{print.SpiderId} total {statistics.Total}, success {statistics.Success}, failure {statistics.Failure}, left {left}");
                    }
                }
                else
                {
                    var log = Encoding.UTF8.GetString(JsonSerializer.SerializeToUtf8Bytes(message));
                    _logger.LogWarning($"Not supported message: {log}");
                }
            };
            await _messageQueue.ConsumeAsync(_consumer, stoppingToken);

            _logger.LogInformation("Statistics service started");
        }
Ejemplo n.º 4
0
        private async Task HandleStatisticsMessageAsync(MessageData <string> message)
        {
            if (message == null)
            {
                _logger.LogWarning("Statistics center receive empty message");
                return;
            }

            switch (message.Type)
            {
            case "Success":
            {
                var ownerId = message.Data;
                await _statisticsStore.IncrementSuccessAsync(ownerId);

                break;
            }

            case "Failed":
            {
                var data = message.Data.Split(',');
                await _statisticsStore.IncrementFailedAsync(data[0], int.Parse(data[1]));

                break;
            }

            case "Start":
            {
                var ownerId = message.Data;
                await _statisticsStore.StartAsync(ownerId);

                break;
            }

            case "Exit":
            {
                var ownerId = message.Data;
                await _statisticsStore.ExitAsync(ownerId);

                break;
            }

            case "Total":
            {
                var data = message.Data.Split(',');
                await _statisticsStore.IncrementTotalAsync(data[0], int.Parse(data[1]));

                break;
            }

            case "DownloadSuccess":
            {
                var data = message.Data.Split(',');
                await _statisticsStore.IncrementDownloadSuccessAsync(data[0], int.Parse(data[1]),
                                                                     long.Parse(data[2]));

                break;
            }

            case "DownloadFailed":
            {
                var data = message.Data.Split(',');
                await _statisticsStore.IncrementDownloadFailedAsync(data[0], int.Parse(data[1]),
                                                                    long.Parse(data[2]));

                break;
            }

            case "Print":
            {
                var ownerId    = message.Data;
                var statistics = await _statisticsStore.GetSpiderStatisticsAsync(ownerId);

                if (statistics != null)
                {
                    _logger.LogInformation(
                        $"{ownerId} total {statistics.Total}, success {statistics.Success}, failed {statistics.Failed}, left {statistics.Total - statistics.Success - statistics.Failed}");
                }

                break;
            }
            }
        }
Ejemplo n.º 5
0
        /// <summary>
        /// 启动统计服务中心
        /// </summary>
        /// <param name="cancellationToken"></param>
        /// <returns></returns>
        /// <exception cref="SpiderException"></exception>
        public async Task StartAsync(CancellationToken cancellationToken)
        {
            if (IsRunning)
            {
                throw new SpiderException("统计中心正在运行中");
            }

            await _statisticsStore.EnsureDatabaseAndTableCreatedAsync();

            _logger.LogInformation("统计中心准备数据库完成");
            _eventBus.Subscribe(Framework.StatisticsServiceTopic, async message =>
            {
                var commandMessage = message.ToCommandMessage();
                if (commandMessage == null)
                {
                    _logger.LogWarning($"接收到非法消息: {message}");
                    return;
                }

                switch (commandMessage.Command)
                {
                case "Success":
                    {
                        var ownerId = commandMessage.Message;
                        await _statisticsStore.IncrementSuccessAsync(ownerId);
                        break;
                    }

                case "Failed":
                    {
                        var data = commandMessage.Message.Split(',');
                        await _statisticsStore.IncrementFailedAsync(data[0], int.Parse(data[1]));
                        break;
                    }

                case "Start":
                    {
                        var ownerId = commandMessage.Message;
                        await _statisticsStore.StartAsync(ownerId);
                        break;
                    }

                case "Exit":
                    {
                        var ownerId = commandMessage.Message;
                        await _statisticsStore.ExitAsync(ownerId);
                        break;
                    }

                case "Total":
                    {
                        var data = commandMessage.Message.Split(',');
                        await _statisticsStore.IncrementTotalAsync(data[0], int.Parse(data[1]));

                        break;
                    }

                case "DownloadSuccess":
                    {
                        var data = commandMessage.Message.Split(',');
                        await _statisticsStore.IncrementDownloadSuccessAsync(data[0], int.Parse(data[1]),
                                                                             long.Parse(data[2]));
                        break;
                    }

                case "DownloadFailed":
                    {
                        var data = commandMessage.Message.Split(',');
                        await _statisticsStore.IncrementDownloadFailedAsync(data[0], int.Parse(data[1]),
                                                                            long.Parse(data[2]));
                        break;
                    }

                case "Print":
                    {
                        var ownerId    = commandMessage.Message;
                        var statistics = await _statisticsStore.GetSpiderStatisticsAsync(ownerId);
                        _logger.LogInformation(
                            $"任务 {ownerId} 总计 {statistics.Total}, 成功 {statistics.Success}, 失败 {statistics.Failed}, 剩余 {statistics.Total - statistics.Success - statistics.Failed}");
                        break;
                    }
                }
            });

            IsRunning = true;
            _logger.LogInformation("统计中心启动");
        }