Exemple #1
0
        public async Task <bool> RemoveContainerAsync(int id)
        {
            try
            {
                var spiderHistory = await _dockerCrawlerService.GetHistory(id);

                if (spiderHistory == null)
                {
                    throw new ApplicationException("Spider history is not exits");
                }

                var containerId = spiderHistory.ContainerId;
                if (string.IsNullOrWhiteSpace(containerId))
                {
                    throw new ApplicationException("Spider container is not exits");
                }
                var client = new DockerClientConfiguration(
                    new Uri(_crawlerOptions.Docker))
                             .CreateClient();
                await client.Containers.StopContainerAsync(containerId, new ContainerStopParameters());

                await client.Containers.RemoveContainerAsync(containerId, new ContainerRemoveParameters());

                await _dockerCrawlerService.UpdateSpiderContainerStatus(spiderHistory.Id, "Removed");

                return(true);
            }
            catch (Exception e)
            {
                _logger.LogException(e, $"关闭失败");
                return(false);
            }
        }
Exemple #2
0
        public async Task ExecuteJob(int spiderId)
        {
            var client = new DockerClientConfiguration(
                new Uri(_crawlerOptions.Docker))
                         .CreateClient();

            try
            {
                var containers = await client.Containers.ListContainersAsync(new ContainersListParameters
                {
                    All     = true,
                    Filters = new Dictionary <string, IDictionary <string, bool> >
                    {
                        {
                            "name", new Dictionary <string, bool>
                            {
                                { "dotnetspider*", true }
                            }
                        }
                    }
                });

                foreach (var container in containers)
                {
                    if (container.State == "exited")
                    {
                        await client.Containers.RemoveContainerAsync(container.ID, new ContainerRemoveParameters());
                    }
                }
            }
            catch { }

            try
            {
                _logger.LogInformation($"触发任务 {spiderId}");

                var spider = await _dockerCrawlerService.Get <SpiderDto>(x => x.Id == spiderId);

                if (spider == null)
                {
                    _logger.LogError($"任务 {spiderId} 不存在");
                    return;
                }

                if (!spider.Enabled)
                {
                    _logger.LogError($"任务 {spiderId} 被禁用");
                    return;
                }


                var batch = Guid.NewGuid().ToString("N");
                var env   = new List <string>((spider.Environment ?? "").Split(new[] { " ", "\n" },
                                                                               StringSplitOptions.RemoveEmptyEntries))
                {
                    $"DOTNET_SPIDER_ID={batch}",
                    $"DOTNET_SPIDER_NAME={spider.Name}"
                };

                var name       = $"dotnetspider-{spider.Id}-{batch}";
                var parameters = new CreateContainerParameters
                {
                    Image  = spider.Image,
                    Name   = name,
                    Labels = new Dictionary <string, string>
                    {
                        { "dotnetspider.spider.id", spider.Id.ToString() },
                        { "dotnetspider.spider.batch", batch },
                        { "dotnetspider.spider.name", spider.Name }
                    },
                    Env        = env,
                    HostConfig = new HostConfig()
                };
                var volumes = new HashSet <string>();
                foreach (var volume in _crawlerOptions.DockerVolumes)
                {
                    volumes.Add(volume);
                }

                var configVolumes = new List <string>((spider.Volume ?? "").Split(new[] { " ", "\n" },
                                                                                  StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()));
                foreach (var volume in configVolumes)
                {
                    volumes.Add(volume);
                }

                parameters.HostConfig.Binds       = volumes.ToList();
                parameters.HostConfig.NetworkMode = _crawlerOptions.NetworkMode ?? "";
                parameters.HostConfig.ExtraHosts  = _crawlerOptions.ExtraHosts.ToList();
                var result = await client.Containers.CreateContainerAsync(parameters);

                if (result.ID == null)
                {
                    _logger.LogError($"创建任务 {spiderId} 实例失败: {string.Join(", ", result.Warnings)}");
                }

                var spiderContainer = new SpiderHistoryDto
                {
                    ContainerId  = result.ID,
                    Batch        = batch,
                    SpiderId     = spider.Id,
                    SpiderName   = spider.Name,
                    Status       = "Created",
                    CreationTime = DateTimeOffset.Now
                };

                spiderContainer = await _dockerCrawlerService.AddSpiderContainer(spiderContainer);

                var startResult =
                    await client.Containers.StartContainerAsync(result.ID, new ContainerStartParameters());

                await _dockerCrawlerService.UpdateSpiderContainerStatus(spiderContainer.Id, startResult? "Success" : "Failed");


                _logger.LogInformation($"触发任务 {spiderId} 完成");
            }
            catch (Exception ex)
            {
                _logger.LogError($"触发任务 {spiderId} 失败: {ex}");
            }
        }