/// <summary> /// 分配下载器代理 /// </summary> /// <param name="allotDownloaderMessage">分配下载器代理的选项</param> /// <returns></returns> protected virtual async Task <bool> AllocateAsync(AllotDownloaderMessage allotDownloaderMessage) { var agents = Agents.Values; if (agents.Count == 0) { Logger.LogInformation($"任务 {allotDownloaderMessage.OwnerId} 未找到可用的下载器代理"); return(false); } // TODO: 实现分配 var allocatedAgents = new[] { agents.First() }; var agentIds = allocatedAgents.Select(x => x.Id).ToArray(); // 保存节点选取信息 await DownloaderAgentStore.AllocateAsync(allotDownloaderMessage.OwnerId, agentIds); // 发送消息让下载代理器分配好下载器 var message = $"|{Framework.AllocateDownloaderCommand}|{JsonConvert.SerializeObject(allotDownloaderMessage)}"; foreach (var agent in agents) { await Mq.PublishAsync(agent.Id, message); } Logger.LogInformation( $"任务 {allotDownloaderMessage.OwnerId} 分配下载代理器成功: {JsonConvert.SerializeObject(allocatedAgents)}"); return(true); }
protected override async Task <DownloaderEntry> CreateDownloaderEntry( AllotDownloaderMessage allotDownloaderMessage) { var downloadEntry = await base.CreateDownloaderEntry(allotDownloaderMessage); downloadEntry.Downloader.Logger = null; return(downloadEntry); }
protected virtual async Task <bool> AllocateAsync(AllotDownloaderMessage allotDownloaderMessage) { List <DownloaderAgent> agents = null; for (int i = 0; i < 50; ++i) { agents = await DownloaderAgentStore.GetAllListAsync(); if (agents.Count <= 0) { Thread.Sleep(100); } else { break; } } if (agents == null) { Logger.LogError("未找到活跃的下载器代理"); return(false); } // 保存节点选取信息 await DownloaderAgentStore.AllocateAsync(allotDownloaderMessage.OwnerId, new[] { agents[0].Id }); Logger.LogInformation($"分配下载器代理成功 OwnerId {allotDownloaderMessage.OwnerId}, Agent {agents[0].Id}"); // 发送消息让下载代理器分配好下载器 var message = $"|{Framework.AllocateDownloaderCommand}|{JsonConvert.SerializeObject(allotDownloaderMessage)}"; foreach (var agent in agents) { await Mq.PublishAsync(agent.Id, message); } return(true); }
/// <summary> /// 单机模式只有一个下载器代理 /// </summary> /// <param name="allotDownloaderMessage">分配下载器代理的选项</param> /// <returns></returns> protected override async Task <bool> AllocateAsync(AllotDownloaderMessage allotDownloaderMessage) { var agent = Agents.Values.FirstOrDefault(); if (agent == null) { Logger.LogInformation($"任务 {allotDownloaderMessage.OwnerId} 未找到可用的下载器代理"); return(false); } // 保存节点选取信息 await DownloaderAgentStore.AllocateAsync(allotDownloaderMessage.OwnerId, new[] { agent.Id }); // 发送消息让下载代理器分配好下载器 var message = $"|{Framework.AllocateDownloaderCommand}|{JsonConvert.SerializeObject(allotDownloaderMessage)}"; await Mq.PublishAsync(agent.Id, message); Logger.LogInformation( $"任务 {allotDownloaderMessage.OwnerId} 分配下载代理器成功: {JsonConvert.SerializeObject(agent)}"); return(true); }
public Task <IDownloader> CreateDownloaderAsync(string agentId, AllotDownloaderMessage allotDownloaderMessage) { IDownloader downloader = null; switch (allotDownloaderMessage.Type) { case DownloaderType.Empty: { downloader = new EmptyDownloader { AgentId = agentId, Logger = _loggerFactory.CreateLogger <ExceptionDownloader>() }; break; } case DownloaderType.Test: { downloader = new TestDownloader { AgentId = agentId, Logger = _loggerFactory.CreateLogger <ExceptionDownloader>() }; break; } case DownloaderType.Exception: { downloader = new ExceptionDownloader { AgentId = agentId, Logger = _loggerFactory.CreateLogger <ExceptionDownloader>() }; break; } case DownloaderType.WebDriver: { throw new NotImplementedException(); } case DownloaderType.HttpClient: { var httpClient = new HttpClientDownloader { AgentId = agentId, UseProxy = allotDownloaderMessage.UseProxy, AllowAutoRedirect = allotDownloaderMessage.AllowAutoRedirect, Timeout = allotDownloaderMessage.Timeout, DecodeHtml = allotDownloaderMessage.DecodeHtml, UseCookies = allotDownloaderMessage.UseCookies, Logger = _loggerFactory.CreateLogger <HttpClientDownloader>(), HttpProxyPool = string.IsNullOrWhiteSpace(_options.ProxySupplyUrl) ? null : new HttpProxyPool(new HttpRowTextProxySupplier(_options.ProxySupplyUrl)), RetryTime = allotDownloaderMessage.RetryTimes }; httpClient.AddCookies(allotDownloaderMessage.Cookies); downloader = httpClient; break; } } return(Task.FromResult(downloader)); }
public abstract Task <bool> AllocateAsync(AllotDownloaderMessage allotDownloaderMessage);
public Task <DownloaderEntry> CreateDownloaderEntryAsync(AllotDownloaderMessage allotDownloaderMessage) { DownloaderEntry downloaderEntry = null; // TODO: 添加其它下载器的分配方法 switch (allotDownloaderMessage.Type) { case DownloaderType.Empty: { downloaderEntry = new DownloaderEntry { LastUsedTime = DateTime.Now, Downloader = new EmptyDownloader() }; break; } case DownloaderType.Test: { downloaderEntry = new DownloaderEntry { LastUsedTime = DateTime.Now, Downloader = new TestDownloader() }; break; } case DownloaderType.Exception: { downloaderEntry = new DownloaderEntry { LastUsedTime = DateTime.Now, Downloader = new ExceptionDownloader() }; break; } case DownloaderType.WebDriver: { throw new NotImplementedException(); } case DownloaderType.HttpClient: { var httpClient = new HttpClientDownloader { UseProxy = allotDownloaderMessage.UseProxy, AllowAutoRedirect = allotDownloaderMessage.AllowAutoRedirect, Timeout = allotDownloaderMessage.Timeout, DecodeHtml = allotDownloaderMessage.DecodeHtml, UseCookies = allotDownloaderMessage.UseCookies }; httpClient.AddCookies(allotDownloaderMessage.Cookies); downloaderEntry = new DownloaderEntry { LastUsedTime = DateTime.Now, Downloader = httpClient }; break; } } return(Task.FromResult(downloaderEntry)); }