/// <summary>
/// 分配下载器代理
/// </summary>
/// <param name="allotDownloaderMessage">分配下载器代理的消息</param>
/// <param name="message">分配下载器代理的消息</param>
/// <returns></returns>
protected virtual async Task <bool> AllocateAsync(AllocateDownloaderMessage allotDownloaderMessage,
string message)
{
var agents = Agents.Values;
// 计算需要分配的个数
var count = allotDownloaderMessage.DownloaderCount >= agents.Count
? agents.Count
: allotDownloaderMessage.DownloaderCount;
var agentIds = agents.OrderBy(_ => Guid.NewGuid()).Take(count).Select(x => x.Id).ToArray();
// Agents 是异步更新的, 因此以最终分配结果来判断是否能够分配正确
if (agentIds.Length == 0)
{
Logger.LogInformation($"任务 {allotDownloaderMessage.OwnerId} 未分配到下载器代理");
return(false);
}
if (agentIds.Length < allotDownloaderMessage.DownloaderCount)
{
Logger.LogWarning($"任务 {allotDownloaderMessage.OwnerId} 未足额分配下载器代理");
}
// 发送消息让下载代理器分配好下载器
var msg =
$"|{Framework.AllocateDownloaderCommand}|{JsonConvert.SerializeObject(allotDownloaderMessage)}";
foreach (var agent in agents)
{
await Mq.PublishAsync(agent.Id, msg);
}
// 保存节点分配信息到数据库
await DownloaderAgentStore.AllocateAsync(allotDownloaderMessage.OwnerId, message, agentIds);
// 更新缓存中的分配信息
AllocatedAgents.AddOrUpdate(allotDownloaderMessage.OwnerId,
new Tuple <AllocateDownloaderMessage, string[]>(allotDownloaderMessage, agentIds), (s, tuple) => tuple);
Logger.LogInformation(
$"任务 {allotDownloaderMessage.OwnerId} 分配下载代理器成功: {JsonConvert.SerializeObject(agentIds)}");
return(true);
}
/// <summary>
/// 分配下载器代理
/// </summary>
/// <param name="allotDownloaderMessage">分配下载器代理的消息</param>
/// <param name="message">分配下载器代理的消息</param>
/// <returns></returns>
protected virtual async Task <bool> AllocateAsync(AllocateDownloaderMessage allotDownloaderMessage,
string message)
{
var agents = Agents.Values;
// Calculate the number of allocations required
var count = allotDownloaderMessage.DownloaderCount >= agents.Count
? agents.Count
: allotDownloaderMessage.DownloaderCount;
var agentIds = agents.OrderBy(_ => Guid.NewGuid()).Take(count).Select(x => x.Id).ToArray();
// Agents are updated asynchronously, so the final allocation result is used to determine whether the allocation is correct.
if (agentIds.Length == 0)
{
Logger.LogInformation($"任务 {allotDownloaderMessage.OwnerId} 未分配到下载器代理");
return(false);
}
if (agentIds.Length < allotDownloaderMessage.DownloaderCount)
{
Logger.LogWarning($"任务 {allotDownloaderMessage.OwnerId} 未足额分配下载器代理");
}
// Send a message to let the download agent allocate the downloader
var msg =
$"|{Framework.AllocateDownloaderCommand}|{JsonConvert.SerializeObject(allotDownloaderMessage)}";
foreach (var agent in agents)
{
await Mq.PublishAsync(agent.Id, msg);
}
// Save node allocation information to the storage
await DownloaderAgentStore.AllocateAsync(allotDownloaderMessage.OwnerId, message, agentIds);
// Update allocation information in the cache
AllocatedAgents.AddOrUpdate(allotDownloaderMessage.OwnerId,
new Tuple <AllocateDownloaderMessage, string[]>(allotDownloaderMessage, agentIds), (s, tuple) => tuple);
Logger.LogInformation(
$"任务 {allotDownloaderMessage.OwnerId} 分配下载代理器成功: {JsonConvert.SerializeObject(agentIds)}");
return(true);
}
/// <summary>
/// 分配下载器代理
/// </summary>
/// <param name="allotDownloaderMessage">分配下载器代理的消息</param>
/// <param name="message">分配下载器代理的消息</param>
/// <returns></returns>
protected override async Task <bool> AllocateAsync(AllocateDownloaderMessage allotDownloaderMessage,
string message)
{
var agent = Agents.Values.FirstOrDefault();
if (agent == null)
{
Logger.LogInformation($"任务 {allotDownloaderMessage.OwnerId} 未找到可用的下载器代理");
return(false);
}
// 保存节点选取信息
await DownloaderAgentStore.AllocateAsync(allotDownloaderMessage.OwnerId, message, new[] { agent.Id });
// 发送消息让下载代理器分配好下载器
var msg =
$"|{Framework.AllocateDownloaderCommand}|{JsonConvert.SerializeObject(allotDownloaderMessage)}";
await Mq.PublishAsync(agent.Id, msg);
Logger.LogInformation(
$"任务 {allotDownloaderMessage.OwnerId} 分配下载代理器成功: {JsonConvert.SerializeObject(agent)}");
return(true);
}
/// <summary>
/// 创建下载器
/// </summary>
/// <param name="agentId">下载器代理标识</param>
/// <param name="allotDownloaderMessage">下载器配置信息</param>
/// <returns></returns>
/// <exception cref="NotImplementedException"></exception>
public Task <IDownloader> CreateDownloaderAsync(string agentId,
AllocateDownloaderMessage allotDownloaderMessage)
{
IDownloader downloader = null;
switch (allotDownloaderMessage.Type)
{
case DownloaderType.Empty:
{
downloader = new EmptyDownloader
{
AgentId = agentId,
Logger = _loggerFactory.CreateLogger <ExceptionDownloader>()
};
break;
}
case DownloaderType.Test:
{
downloader = new TestDownloader
{
AgentId = agentId,
Logger = _loggerFactory.CreateLogger <ExceptionDownloader>()
};
break;
}
case DownloaderType.Exception:
{
downloader = new ExceptionDownloader
{
AgentId = agentId,
Logger = _loggerFactory.CreateLogger <ExceptionDownloader>()
};
break;
}
case DownloaderType.WebDriver:
{
throw new NotImplementedException();
}
case DownloaderType.HttpClient:
{
var httpClient = new HttpClientDownloader
{
AgentId = agentId,
UseProxy = allotDownloaderMessage.UseProxy,
AllowAutoRedirect = allotDownloaderMessage.AllowAutoRedirect,
Timeout = allotDownloaderMessage.Timeout,
DecodeHtml = allotDownloaderMessage.DecodeHtml,
UseCookies = allotDownloaderMessage.UseCookies,
Logger = _loggerFactory.CreateLogger <HttpClientDownloader>(),
HttpProxyPool = string.IsNullOrWhiteSpace(_options.ProxySupplyUrl)
? null
: new HttpProxyPool(new HttpRowTextProxySupplier(_options.ProxySupplyUrl))
{
ProxyValidator = _proxyValidator
},
RetryTime = allotDownloaderMessage.RetryTimes
};
httpClient.AddCookies(allotDownloaderMessage.Cookies);
downloader = httpClient;
break;
}
}
return(Task.FromResult(downloader));
}
