/// <summary>
/// 分配下载器代理
/// </summary>
/// <param name="allotDownloaderMessage">分配下载器代理的选项</param>
/// <returns></returns>
protected virtual async Task <bool> AllocateAsync(AllotDownloaderMessage allotDownloaderMessage)
{
var agents = Agents.Values;
if (agents.Count == 0)
{
Logger.LogInformation($"任务 {allotDownloaderMessage.OwnerId} 未找到可用的下载器代理");
return(false);
}
// TODO: 实现分配
var allocatedAgents = new[] { agents.First() };
var agentIds = allocatedAgents.Select(x => x.Id).ToArray();
// 保存节点选取信息
await DownloaderAgentStore.AllocateAsync(allotDownloaderMessage.OwnerId, agentIds);
// 发送消息让下载代理器分配好下载器
var message =
$"|{Framework.AllocateDownloaderCommand}|{JsonConvert.SerializeObject(allotDownloaderMessage)}";
foreach (var agent in agents)
{
await Mq.PublishAsync(agent.Id, message);
}
Logger.LogInformation(
$"任务 {allotDownloaderMessage.OwnerId} 分配下载代理器成功: {JsonConvert.SerializeObject(allocatedAgents)}");
return(true);
}
protected override async Task <DownloaderEntry> CreateDownloaderEntry(
AllotDownloaderMessage allotDownloaderMessage)
{
var downloadEntry = await base.CreateDownloaderEntry(allotDownloaderMessage);
downloadEntry.Downloader.Logger = null;
return(downloadEntry);
}
protected virtual async Task <bool> AllocateAsync(AllotDownloaderMessage allotDownloaderMessage)
{
List <DownloaderAgent> agents = null;
for (int i = 0; i < 50; ++i)
{
agents = await DownloaderAgentStore.GetAllListAsync();
if (agents.Count <= 0)
{
Thread.Sleep(100);
}
else
{
break;
}
}
if (agents == null)
{
Logger.LogError("未找到活跃的下载器代理");
return(false);
}
// 保存节点选取信息
await DownloaderAgentStore.AllocateAsync(allotDownloaderMessage.OwnerId, new[] { agents[0].Id });
Logger.LogInformation($"分配下载器代理成功 OwnerId {allotDownloaderMessage.OwnerId}, Agent {agents[0].Id}");
// 发送消息让下载代理器分配好下载器
var message =
$"|{Framework.AllocateDownloaderCommand}|{JsonConvert.SerializeObject(allotDownloaderMessage)}";
foreach (var agent in agents)
{
await Mq.PublishAsync(agent.Id, message);
}
return(true);
}
/// <summary>
/// 单机模式只有一个下载器代理
/// </summary>
/// <param name="allotDownloaderMessage">分配下载器代理的选项</param>
/// <returns></returns>
protected override async Task <bool> AllocateAsync(AllotDownloaderMessage allotDownloaderMessage)
{
var agent = Agents.Values.FirstOrDefault();
if (agent == null)
{
Logger.LogInformation($"任务 {allotDownloaderMessage.OwnerId} 未找到可用的下载器代理");
return(false);
}
// 保存节点选取信息
await DownloaderAgentStore.AllocateAsync(allotDownloaderMessage.OwnerId, new[] { agent.Id });
// 发送消息让下载代理器分配好下载器
var message =
$"|{Framework.AllocateDownloaderCommand}|{JsonConvert.SerializeObject(allotDownloaderMessage)}";
await Mq.PublishAsync(agent.Id, message);
Logger.LogInformation(
$"任务 {allotDownloaderMessage.OwnerId} 分配下载代理器成功: {JsonConvert.SerializeObject(agent)}");
return(true);
}
public Task <IDownloader> CreateDownloaderAsync(string agentId,
AllotDownloaderMessage allotDownloaderMessage)
{
IDownloader downloader = null;
switch (allotDownloaderMessage.Type)
{
case DownloaderType.Empty:
{
downloader = new EmptyDownloader
{
AgentId = agentId,
Logger = _loggerFactory.CreateLogger <ExceptionDownloader>()
};
break;
}
case DownloaderType.Test:
{
downloader = new TestDownloader
{
AgentId = agentId,
Logger = _loggerFactory.CreateLogger <ExceptionDownloader>()
};
break;
}
case DownloaderType.Exception:
{
downloader = new ExceptionDownloader
{
AgentId = agentId,
Logger = _loggerFactory.CreateLogger <ExceptionDownloader>()
};
break;
}
case DownloaderType.WebDriver:
{
throw new NotImplementedException();
}
case DownloaderType.HttpClient:
{
var httpClient = new HttpClientDownloader
{
AgentId = agentId,
UseProxy = allotDownloaderMessage.UseProxy,
AllowAutoRedirect = allotDownloaderMessage.AllowAutoRedirect,
Timeout = allotDownloaderMessage.Timeout,
DecodeHtml = allotDownloaderMessage.DecodeHtml,
UseCookies = allotDownloaderMessage.UseCookies,
Logger = _loggerFactory.CreateLogger <HttpClientDownloader>(),
HttpProxyPool = string.IsNullOrWhiteSpace(_options.ProxySupplyUrl)
? null
: new HttpProxyPool(new HttpRowTextProxySupplier(_options.ProxySupplyUrl)),
RetryTime = allotDownloaderMessage.RetryTimes
};
httpClient.AddCookies(allotDownloaderMessage.Cookies);
downloader = httpClient;
break;
}
}
return(Task.FromResult(downloader));
}
public abstract Task <bool> AllocateAsync(AllotDownloaderMessage allotDownloaderMessage);
public Task <DownloaderEntry> CreateDownloaderEntryAsync(AllotDownloaderMessage allotDownloaderMessage)
{
DownloaderEntry downloaderEntry = null;
// TODO: 添加其它下载器的分配方法
switch (allotDownloaderMessage.Type)
{
case DownloaderType.Empty:
{
downloaderEntry = new DownloaderEntry
{
LastUsedTime = DateTime.Now,
Downloader = new EmptyDownloader()
};
break;
}
case DownloaderType.Test:
{
downloaderEntry = new DownloaderEntry
{
LastUsedTime = DateTime.Now,
Downloader = new TestDownloader()
};
break;
}
case DownloaderType.Exception:
{
downloaderEntry = new DownloaderEntry
{
LastUsedTime = DateTime.Now,
Downloader = new ExceptionDownloader()
};
break;
}
case DownloaderType.WebDriver:
{
throw new NotImplementedException();
}
case DownloaderType.HttpClient:
{
var httpClient = new HttpClientDownloader
{
UseProxy = allotDownloaderMessage.UseProxy,
AllowAutoRedirect = allotDownloaderMessage.AllowAutoRedirect,
Timeout = allotDownloaderMessage.Timeout,
DecodeHtml = allotDownloaderMessage.DecodeHtml,
UseCookies = allotDownloaderMessage.UseCookies
};
httpClient.AddCookies(allotDownloaderMessage.Cookies);
downloaderEntry = new DownloaderEntry
{
LastUsedTime = DateTime.Now,
Downloader = httpClient
};
break;
}
}
return(Task.FromResult(downloaderEntry));
}
