//public async Task<DockerRepositoryDto> AddRepository(DockerRepositoryDto repository) //{ // var dockerRepository = Mapper.Map<DockerRepository>(repository); // dockerRepository = await Daos.CurrentDao.AddOrUpdate<DockerRepository, int>(dockerRepository); // return Mapper.Map<DockerRepositoryDto>(dockerRepository); //} public async Task <SpiderHistoryDto> AddSpiderContainer(SpiderHistoryDto spiderContainer) { var container = Mapper.Map <SpiderHistory>(spiderContainer); container = await Daos.CurrentDao.AddOrUpdate <SpiderHistory, int>(container); return(Mapper.Map <SpiderHistoryDto>(container)); }
public async Task ExecuteJob(int spiderId) { var client = new DockerClientConfiguration( new Uri(_crawlerOptions.Docker)) .CreateClient(); try { var containers = await client.Containers.ListContainersAsync(new ContainersListParameters { All = true, Filters = new Dictionary <string, IDictionary <string, bool> > { { "name", new Dictionary <string, bool> { { "dotnetspider*", true } } } } }); foreach (var container in containers) { if (container.State == "exited") { await client.Containers.RemoveContainerAsync(container.ID, new ContainerRemoveParameters()); } } } catch { } try { _logger.LogInformation($"触发任务 {spiderId}"); var spider = await _dockerCrawlerService.Get <SpiderDto>(x => x.Id == spiderId); if (spider == null) { _logger.LogError($"任务 {spiderId} 不存在"); return; } if (!spider.Enabled) { _logger.LogError($"任务 {spiderId} 被禁用"); return; } var batch = Guid.NewGuid().ToString("N"); var env = new List <string>((spider.Environment ?? "").Split(new[] { " ", "\n" }, StringSplitOptions.RemoveEmptyEntries)) { $"DOTNET_SPIDER_ID={batch}", $"DOTNET_SPIDER_NAME={spider.Name}" }; var name = $"dotnetspider-{spider.Id}-{batch}"; var parameters = new CreateContainerParameters { Image = spider.Image, Name = name, Labels = new Dictionary <string, string> { { "dotnetspider.spider.id", spider.Id.ToString() }, { "dotnetspider.spider.batch", batch }, { "dotnetspider.spider.name", spider.Name } }, Env = env, HostConfig = new HostConfig() }; var volumes = new HashSet <string>(); foreach (var volume in _crawlerOptions.DockerVolumes) { volumes.Add(volume); } var configVolumes = new List <string>((spider.Volume ?? "").Split(new[] { " ", "\n" }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim())); foreach (var volume in configVolumes) { volumes.Add(volume); } parameters.HostConfig.Binds = volumes.ToList(); parameters.HostConfig.NetworkMode = _crawlerOptions.NetworkMode ?? ""; parameters.HostConfig.ExtraHosts = _crawlerOptions.ExtraHosts.ToList(); var result = await client.Containers.CreateContainerAsync(parameters); if (result.ID == null) { _logger.LogError($"创建任务 {spiderId} 实例失败: {string.Join(", ", result.Warnings)}"); } var spiderContainer = new SpiderHistoryDto { ContainerId = result.ID, Batch = batch, SpiderId = spider.Id, SpiderName = spider.Name, Status = "Created", CreationTime = DateTimeOffset.Now }; spiderContainer = await _dockerCrawlerService.AddSpiderContainer(spiderContainer); var startResult = await client.Containers.StartContainerAsync(result.ID, new ContainerStartParameters()); await _dockerCrawlerService.UpdateSpiderContainerStatus(spiderContainer.Id, startResult? "Success" : "Failed"); _logger.LogInformation($"触发任务 {spiderId} 完成"); } catch (Exception ex) { _logger.LogError($"触发任务 {spiderId} 失败: {ex}"); } }