public async Task <bool> UpdateAsync(int id, SpiderDto viewModel)
{
if (ModelState.IsValid)
{
var spider = await _dockerCrawlerService.Get <SpiderDto>(x => x.Id == id);
if (spider == null)
{
throw new ApplicationException($"Spider {id} exists");
}
var exists = await _dockerCrawlerService.CountAsync <SpiderDto>(x =>
x.Name == viewModel.Name && x.Id != id);
if (exists > 0)
{
ModelState.AddModelError("Name", "名称已经存在");
}
try
{
CrontabSchedule.Parse(viewModel.Cron);
}
catch
{
ModelState.AddModelError("Cron", "Cron 表达式不正确");
}
spider.Name = viewModel.Name;
spider.Cron = viewModel.Cron;
spider.Image = viewModel.Image;
spider.Volume = viewModel.Volume;
spider.Environment = viewModel.Environment;
spider.LastModificationTime = DateTimeOffset.Now;
await _dockerCrawlerService.AddOrUpdate <SpiderDto, int>(spider);
_recurringJobManager.RemoveIfExists(spider.Name);
await ScheduleJobAsync(spider);
return(true);
}
else
{
throw new ApplicationException("ModelState is invalid");
}
}
public async Task ExecuteJob(int spiderId)
{
var client = new DockerClientConfiguration(
new Uri(_crawlerOptions.Docker))
.CreateClient();
try
{
var containers = await client.Containers.ListContainersAsync(new ContainersListParameters
{
All = true,
Filters = new Dictionary <string, IDictionary <string, bool> >
{
{
"name", new Dictionary <string, bool>
{
{ "dotnetspider*", true }
}
}
}
});
foreach (var container in containers)
{
if (container.State == "exited")
{
await client.Containers.RemoveContainerAsync(container.ID, new ContainerRemoveParameters());
}
}
}
catch { }
try
{
_logger.LogInformation($"触发任务 {spiderId}");
var spider = await _dockerCrawlerService.Get <SpiderDto>(x => x.Id == spiderId);
if (spider == null)
{
_logger.LogError($"任务 {spiderId} 不存在");
return;
}
if (!spider.Enabled)
{
_logger.LogError($"任务 {spiderId} 被禁用");
return;
}
var batch = Guid.NewGuid().ToString("N");
var env = new List <string>((spider.Environment ?? "").Split(new[] { " ", "\n" },
StringSplitOptions.RemoveEmptyEntries))
{
$"DOTNET_SPIDER_ID={batch}",
$"DOTNET_SPIDER_NAME={spider.Name}"
};
var name = $"dotnetspider-{spider.Id}-{batch}";
var parameters = new CreateContainerParameters
{
Image = spider.Image,
Name = name,
Labels = new Dictionary <string, string>
{
{ "dotnetspider.spider.id", spider.Id.ToString() },
{ "dotnetspider.spider.batch", batch },
{ "dotnetspider.spider.name", spider.Name }
},
Env = env,
HostConfig = new HostConfig()
};
var volumes = new HashSet <string>();
foreach (var volume in _crawlerOptions.DockerVolumes)
{
volumes.Add(volume);
}
var configVolumes = new List <string>((spider.Volume ?? "").Split(new[] { " ", "\n" },
StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()));
foreach (var volume in configVolumes)
{
volumes.Add(volume);
}
parameters.HostConfig.Binds = volumes.ToList();
parameters.HostConfig.NetworkMode = _crawlerOptions.NetworkMode ?? "";
parameters.HostConfig.ExtraHosts = _crawlerOptions.ExtraHosts.ToList();
var result = await client.Containers.CreateContainerAsync(parameters);
if (result.ID == null)
{
_logger.LogError($"创建任务 {spiderId} 实例失败: {string.Join(", ", result.Warnings)}");
}
var spiderContainer = new SpiderHistoryDto
{
ContainerId = result.ID,
Batch = batch,
SpiderId = spider.Id,
SpiderName = spider.Name,
Status = "Created",
CreationTime = DateTimeOffset.Now
};
spiderContainer = await _dockerCrawlerService.AddSpiderContainer(spiderContainer);
var startResult =
await client.Containers.StartContainerAsync(result.ID, new ContainerStartParameters());
await _dockerCrawlerService.UpdateSpiderContainerStatus(spiderContainer.Id, startResult? "Success" : "Failed");
_logger.LogInformation($"触发任务 {spiderId} 完成");
}
catch (Exception ex)
{
_logger.LogError($"触发任务 {spiderId} 失败: {ex}");
}
}
