Beispiel #1
0
 private void SubscribeDynamicMessage()
 {
     if (_dmq == null)
     {
         return;
     }
     while (true)
     {
         try
         {
             _dmq.Subscribe(_options.AgentId, HandleDynamicMessage);
             Logger.LogInformation($"订阅节点 {_options.AgentId} 推送请求成功");
             return;
         }
         catch (Exception e)
         {
             Logger.LogError($"订阅节点 {_options.AgentId} 推送请求结果失败: {e.Message}");
             Thread.Sleep(1000);
         }
     }
 }
Beispiel #2
0
        /// <summary>
        /// 启动下载中心
        /// </summary>
        /// <param name="cancellationToken"></param>
        /// <returns></returns>
        /// <exception cref="SpiderException"></exception>
        public async Task StartAsync(CancellationToken cancellationToken)
        {
            if (_isRunning)
            {
                throw new SpiderException("下载中心正在运行中");
            }

            await DownloaderAgentStore.EnsureDatabaseAndTableCreatedAsync();

            await SyncDownloaderAgentData();

            StartSyncDownloaderAgentDataService().ConfigureAwait(false).GetAwaiter();

            Mq.Subscribe(Framework.DownloaderCenterTopic, async message =>
            {
                var commandMessage = message.ToCommandMessage();
                if (commandMessage == null)
                {
                    Logger.LogWarning($"接收到非法消息: {message}");
                    return;
                }

                switch (commandMessage.Command)
                {
                case Framework.RegisterCommand:
                    {
                        // 此处不考虑消息的超时,一是因为节点数量不会很多,二是因为超时的可以释放掉
                        // The timeout of the message is not considered here, because the number of nodes is not much, and the second is because the timeout can be released.
                        var agent = JsonConvert.DeserializeObject <DownloaderAgent>(commandMessage.Message);
                        if (agent != null)
                        {
                            Agents.AddOrUpdate(agent.Id, x => agent, (s, a) =>
                            {
                                a.LastModificationTime = agent.LastModificationTime;
                                return(a);
                            });
                            await DownloaderAgentStore.RegisterAsync(agent);
                            Logger.LogInformation($"Register the Downloader Agent {agent.Id} successfully");
                        }
                        else
                        {
                            Logger.LogError($"Register Downloader Agent Message Is Incorrect: {commandMessage.Message}");
                        }

                        break;
                    }

                case Framework.HeartbeatCommand:
                    {
                        var heartbeat = JsonConvert.DeserializeObject <DownloaderAgentHeartbeat>(commandMessage.Message);
                        if (heartbeat != null)
                        {
                            if ((DateTime.Now - heartbeat.CreationTime).TotalSeconds < Options.MessageExpiredTime)
                            {
                                if (Agents.ContainsKey(heartbeat.AgentId))
                                {
                                    Agents[heartbeat.AgentId].RefreshLastModificationTime();
                                }

                                await DownloaderAgentStore.HeartbeatAsync(heartbeat);
                                Logger.LogDebug($"Downloader Agent {heartbeat.AgentId} updates heartbeat success");
                            }
                            else
                            {
                                Logger.LogWarning($"Downloader Agent {heartbeat.AgentId} update Heartbeat Period");
                            }
                        }
                        else
                        {
                            Logger.LogError($"Downloader Agent heartbeat information is incorrect: {commandMessage.Message}");
                        }

                        break;
                    }

                case Framework.AllocateDownloaderCommand:
                    {
                        var options = JsonConvert.DeserializeObject <AllocateDownloaderMessage>(commandMessage.Message);
                        if (options != null)
                        {
                            if ((DateTime.Now - options.CreationTime).TotalSeconds < Options.MessageExpiredTime)
                            {
                                await AllocateAsync(options, commandMessage.Message);
                            }
                            else
                            {
                                Logger.LogWarning($"任务 {options.OwnerId} 分配下载代理器过期");
                            }
                        }
                        else
                        {
                            Logger.LogError($"分配下载代理器过期信息不正确: {commandMessage.Message}");
                        }

                        break;
                    }

                case Framework.DownloadCommand:
                    {
                        var requests = JsonConvert.DeserializeObject <Request[]>(commandMessage.Message);
                        if (requests != null)
                        {
                            requests = requests.Where(x =>
                                                      (DateTime.Now - x.CreationTime).TotalSeconds <= Options.MessageExpiredTime).ToArray();
                            if (requests.Length > 0)
                            {
                                var ownerId = requests[0].OwnerId;
                                foreach (var request in requests)
                                {
                                    request.CreationTime = DateTime.Now;
                                }

                                await EnqueueRequests(ownerId, requests);
                                Logger.LogDebug($"任务 {ownerId} 下载请求分发成功");
                            }
                        }
                        else
                        {
                            Logger.LogError($"任务请求信息不正确: {commandMessage.Message}");
                        }

                        break;
                    }
                }
            });
            if (Dmq != null)
            {
                Dmq.Subscribe(Framework.DownloaderCenterTopic, async(cmd, message) =>
                {
                    //var commandMessage = message.ToCommandMessage();
                    if (message == null)
                    {
                        Logger.LogWarning($"接收到非法消息: {cmd}");
                        return;
                    }

                    switch (cmd)
                    {
                    case Framework.DownloadCommand:
                        {
                            var requests = message as Request[];                                    // JsonConvert.DeserializeObject<Request[]>(commandMessage.Message);
                            if (requests == null)
                            {
                                var request = message as Request;
                                if (request != null)
                                {
                                    requests = new Request[] { request }
                                }
                                ;
                            }
                            if (requests != null)
                            {
                                requests = requests.Where(x =>
                                                          (DateTime.Now - x.CreationTime).TotalSeconds <= Options.MessageExpiredTime).ToArray();
                                if (requests.Length > 0)
                                {
                                    var ownerId = requests[0].OwnerId;
                                    foreach (var request in requests)
                                    {
                                        request.CreationTime = DateTime.Now;
                                    }

                                    await EnqueueRequests(ownerId, requests);
                                    Logger.LogDebug($"任务 {ownerId} 下载请求分发成功");
                                }
                            }
                            else
                            {
                                Logger.LogError($"任务请求信息不正确: message");
                            }

                            break;
                        }
                    }
                });
            }
            Logger.LogInformation("下载中心启动完毕");
        }