private void SubscribeDynamicMessage()
{
if (_dmq == null)
{
return;
}
while (true)
{
try
{
_dmq.Subscribe(_options.AgentId, HandleDynamicMessage);
Logger.LogInformation($"订阅节点 {_options.AgentId} 推送请求成功");
return;
}
catch (Exception e)
{
Logger.LogError($"订阅节点 {_options.AgentId} 推送请求结果失败: {e.Message}");
Thread.Sleep(1000);
}
}
}
/// <summary>
/// 启动下载中心
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <exception cref="SpiderException"></exception>
public async Task StartAsync(CancellationToken cancellationToken)
{
if (_isRunning)
{
throw new SpiderException("下载中心正在运行中");
}
await DownloaderAgentStore.EnsureDatabaseAndTableCreatedAsync();
await SyncDownloaderAgentData();
StartSyncDownloaderAgentDataService().ConfigureAwait(false).GetAwaiter();
Mq.Subscribe(Framework.DownloaderCenterTopic, async message =>
{
var commandMessage = message.ToCommandMessage();
if (commandMessage == null)
{
Logger.LogWarning($"接收到非法消息: {message}");
return;
}
switch (commandMessage.Command)
{
case Framework.RegisterCommand:
{
// 此处不考虑消息的超时，一是因为节点数量不会很多，二是因为超时的可以释放掉
// The timeout of the message is not considered here, because the number of nodes is not much, and the second is because the timeout can be released.
var agent = JsonConvert.DeserializeObject <DownloaderAgent>(commandMessage.Message);
if (agent != null)
{
Agents.AddOrUpdate(agent.Id, x => agent, (s, a) =>
{
a.LastModificationTime = agent.LastModificationTime;
return(a);
});
await DownloaderAgentStore.RegisterAsync(agent);
Logger.LogInformation($"Register the Downloader Agent {agent.Id} successfully");
}
else
{
Logger.LogError($"Register Downloader Agent Message Is Incorrect: {commandMessage.Message}");
}
break;
}
case Framework.HeartbeatCommand:
{
var heartbeat = JsonConvert.DeserializeObject <DownloaderAgentHeartbeat>(commandMessage.Message);
if (heartbeat != null)
{
if ((DateTime.Now - heartbeat.CreationTime).TotalSeconds < Options.MessageExpiredTime)
{
if (Agents.ContainsKey(heartbeat.AgentId))
{
Agents[heartbeat.AgentId].RefreshLastModificationTime();
}
await DownloaderAgentStore.HeartbeatAsync(heartbeat);
Logger.LogDebug($"Downloader Agent {heartbeat.AgentId} updates heartbeat success");
}
else
{
Logger.LogWarning($"Downloader Agent {heartbeat.AgentId} update Heartbeat Period");
}
}
else
{
Logger.LogError($"Downloader Agent heartbeat information is incorrect: {commandMessage.Message}");
}
break;
}
case Framework.AllocateDownloaderCommand:
{
var options = JsonConvert.DeserializeObject <AllocateDownloaderMessage>(commandMessage.Message);
if (options != null)
{
if ((DateTime.Now - options.CreationTime).TotalSeconds < Options.MessageExpiredTime)
{
await AllocateAsync(options, commandMessage.Message);
}
else
{
Logger.LogWarning($"任务 {options.OwnerId} 分配下载代理器过期");
}
}
else
{
Logger.LogError($"分配下载代理器过期信息不正确: {commandMessage.Message}");
}
break;
}
case Framework.DownloadCommand:
{
var requests = JsonConvert.DeserializeObject <Request[]>(commandMessage.Message);
if (requests != null)
{
requests = requests.Where(x =>
(DateTime.Now - x.CreationTime).TotalSeconds <= Options.MessageExpiredTime).ToArray();
if (requests.Length > 0)
{
var ownerId = requests[0].OwnerId;
foreach (var request in requests)
{
request.CreationTime = DateTime.Now;
}
await EnqueueRequests(ownerId, requests);
Logger.LogDebug($"任务 {ownerId} 下载请求分发成功");
}
}
else
{
Logger.LogError($"任务请求信息不正确: {commandMessage.Message}");
}
break;
}
}
});
if (Dmq != null)
{
Dmq.Subscribe(Framework.DownloaderCenterTopic, async(cmd, message) =>
{
//var commandMessage = message.ToCommandMessage();
if (message == null)
{
Logger.LogWarning($"接收到非法消息: {cmd}");
return;
}
switch (cmd)
{
case Framework.DownloadCommand:
{
var requests = message as Request[]; // JsonConvert.DeserializeObject<Request[]>(commandMessage.Message);
if (requests == null)
{
var request = message as Request;
if (request != null)
{
requests = new Request[] { request }
}
;
}
if (requests != null)
{
requests = requests.Where(x =>
(DateTime.Now - x.CreationTime).TotalSeconds <= Options.MessageExpiredTime).ToArray();
if (requests.Length > 0)
{
var ownerId = requests[0].OwnerId;
foreach (var request in requests)
{
request.CreationTime = DateTime.Now;
}
await EnqueueRequests(ownerId, requests);
Logger.LogDebug($"任务 {ownerId} 下载请求分发成功");
}
}
else
{
Logger.LogError($"任务请求信息不正确: message");
}
break;
}
}
});
}
Logger.LogInformation("下载中心启动完毕");
}
