private void SubscribeDynamicMessage() { if (_dmq == null) { return; } while (true) { try { _dmq.Subscribe(_options.AgentId, HandleDynamicMessage); Logger.LogInformation($"订阅节点 {_options.AgentId} 推送请求成功"); return; } catch (Exception e) { Logger.LogError($"订阅节点 {_options.AgentId} 推送请求结果失败: {e.Message}"); Thread.Sleep(1000); } } }
/// <summary> /// 启动下载中心 /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> /// <exception cref="SpiderException"></exception> public async Task StartAsync(CancellationToken cancellationToken) { if (_isRunning) { throw new SpiderException("下载中心正在运行中"); } await DownloaderAgentStore.EnsureDatabaseAndTableCreatedAsync(); await SyncDownloaderAgentData(); StartSyncDownloaderAgentDataService().ConfigureAwait(false).GetAwaiter(); Mq.Subscribe(Framework.DownloaderCenterTopic, async message => { var commandMessage = message.ToCommandMessage(); if (commandMessage == null) { Logger.LogWarning($"接收到非法消息: {message}"); return; } switch (commandMessage.Command) { case Framework.RegisterCommand: { // 此处不考虑消息的超时,一是因为节点数量不会很多,二是因为超时的可以释放掉 // The timeout of the message is not considered here, because the number of nodes is not much, and the second is because the timeout can be released. var agent = JsonConvert.DeserializeObject <DownloaderAgent>(commandMessage.Message); if (agent != null) { Agents.AddOrUpdate(agent.Id, x => agent, (s, a) => { a.LastModificationTime = agent.LastModificationTime; return(a); }); await DownloaderAgentStore.RegisterAsync(agent); Logger.LogInformation($"Register the Downloader Agent {agent.Id} successfully"); } else { Logger.LogError($"Register Downloader Agent Message Is Incorrect: {commandMessage.Message}"); } break; } case Framework.HeartbeatCommand: { var heartbeat = JsonConvert.DeserializeObject <DownloaderAgentHeartbeat>(commandMessage.Message); if (heartbeat != null) { if ((DateTime.Now - heartbeat.CreationTime).TotalSeconds < Options.MessageExpiredTime) { if (Agents.ContainsKey(heartbeat.AgentId)) { Agents[heartbeat.AgentId].RefreshLastModificationTime(); } await DownloaderAgentStore.HeartbeatAsync(heartbeat); Logger.LogDebug($"Downloader Agent {heartbeat.AgentId} updates heartbeat success"); } else { Logger.LogWarning($"Downloader Agent {heartbeat.AgentId} update Heartbeat Period"); } } else { Logger.LogError($"Downloader Agent heartbeat information is incorrect: {commandMessage.Message}"); } break; } case Framework.AllocateDownloaderCommand: { var options = JsonConvert.DeserializeObject <AllocateDownloaderMessage>(commandMessage.Message); if (options != null) { if ((DateTime.Now - options.CreationTime).TotalSeconds < Options.MessageExpiredTime) { await AllocateAsync(options, commandMessage.Message); } else { Logger.LogWarning($"任务 {options.OwnerId} 分配下载代理器过期"); } } else { Logger.LogError($"分配下载代理器过期信息不正确: {commandMessage.Message}"); } break; } case Framework.DownloadCommand: { var requests = JsonConvert.DeserializeObject <Request[]>(commandMessage.Message); if (requests != null) { requests = requests.Where(x => (DateTime.Now - x.CreationTime).TotalSeconds <= Options.MessageExpiredTime).ToArray(); if (requests.Length > 0) { var ownerId = requests[0].OwnerId; foreach (var request in requests) { request.CreationTime = DateTime.Now; } await EnqueueRequests(ownerId, requests); Logger.LogDebug($"任务 {ownerId} 下载请求分发成功"); } } else { Logger.LogError($"任务请求信息不正确: {commandMessage.Message}"); } break; } } }); if (Dmq != null) { Dmq.Subscribe(Framework.DownloaderCenterTopic, async(cmd, message) => { //var commandMessage = message.ToCommandMessage(); if (message == null) { Logger.LogWarning($"接收到非法消息: {cmd}"); return; } switch (cmd) { case Framework.DownloadCommand: { var requests = message as Request[]; // JsonConvert.DeserializeObject<Request[]>(commandMessage.Message); if (requests == null) { var request = message as Request; if (request != null) { requests = new Request[] { request } } ; } if (requests != null) { requests = requests.Where(x => (DateTime.Now - x.CreationTime).TotalSeconds <= Options.MessageExpiredTime).ToArray(); if (requests.Length > 0) { var ownerId = requests[0].OwnerId; foreach (var request in requests) { request.CreationTime = DateTime.Now; } await EnqueueRequests(ownerId, requests); Logger.LogDebug($"任务 {ownerId} 下载请求分发成功"); } } else { Logger.LogError($"任务请求信息不正确: message"); } break; } } }); } Logger.LogInformation("下载中心启动完毕"); }