public static async Task RunAsync(PortalOptions options, PortalDbContext dbContext, int jobId)
{
var spider = await dbContext.Spiders.FirstOrDefaultAsync(x => x.Id == jobId);
if (spider == null)
{
throw new Exception($"任务 {jobId} 不存在");
}
DockerClient client = new DockerClientConfiguration(
new Uri("http://localhost:2376"))
.CreateClient();
var env = new List <string>((spider.Environment ?? "").Split(new[] { " " },
StringSplitOptions.RemoveEmptyEntries))
{
$"id={spider.Id}",
$"type={spider.Type}",
$"name={spider.Name}"
};
var image = $"{spider.Registry}{spider.Repository}:{spider.Tag}";
var result = await client.Containers.CreateContainerAsync(new CreateContainerParameters
{
Image = image,
Name = $"dotnetspider-{spider.Id}",
Labels = new Dictionary <string, string>
{
{ "dotnetspider.spider.id", spider.Id.ToString() },
{ "dotnetspider.spider.type", spider.Type },
{ "dotnetspider.spider.name", spider.Name }
},
Volumes = new Dictionary <string, EmptyStruct>
{
{
options.DockerVolumes, new EmptyStruct()
}
},
Env = env
});
if (result.ID == null)
{
throw new Exception($"创建任务 {jobId} 实例失败: {string.Join(", ", result.Warnings)}");
}
var spiderContainer = new SpiderContainer
{
ContainerId = result.ID,
SpiderId = spider.Id,
Status = "OK"
};
dbContext.SpiderContainers.Add(spiderContainer);
await dbContext.SaveChangesAsync();
}
public static async Task RunAsync(PortalOptions options, PortalDbContext dbContext, int jobId)
{
var spider = await dbContext.Spiders.FirstOrDefaultAsync(x => x.Id == jobId);
if (spider == null)
{
throw new Exception($"任务 {jobId} 不存在");
}
var docker = new DockerClient.DockerClient(new Uri(options.Docker));
bool exists = await docker.ExistsAsync(new
{
label = new[] { $"dotnetspider.spider.id={spider.Id}" }
});
if (exists)
{
throw new Exception($"任务 {spider.Id} 正在运行");
}
var env = new List <string>((spider.Environment ?? "").Split(new[] { " " },
StringSplitOptions.RemoveEmptyEntries))
{
$"id={spider.Id}",
$"type={spider.Type}",
$"name={spider.Name}"
};
var result = await docker.PullAsync(spider.Image);
if (!result.Success)
{
throw new Exception($"接取镜像 {spider.Image} 失败: {result.Message}");
}
result = await docker.CreateAsync(spider.Image,
env.ToArray(),
new Dictionary <string, object>
{
{ "id", spider.Id.ToString() },
{ "type", spider.Type },
{ "name", spider.Name }
}, new[] { options.DockerVolumes });
if (!result.Success)
{
throw new Exception($"创建任务 {jobId} 实例失败: {result.Message}");
}
var spiderContainer = new SpiderContainer
{
ContainerId = result.Id,
SpiderId = spider.Id,
Status = "Creating"
};
result = await docker.StartAsync(spiderContainer.ContainerId);
if (result.Success)
{
spiderContainer.Status = "OK";
}
dbContext.SpiderContainers.Add(spiderContainer);
await dbContext.SaveChangesAsync();
}
public async Task Execute(IJobExecutionContext context)
{
var jobId = context.JobDetail.Key.Name;
using (var scope = ServiceProvider.Instance.CreateScope())
{
var services = scope.ServiceProvider;
var logger = services.GetRequiredService <ILogger <TriggerJob> >();
try
{
var options = services.GetRequiredService <PortalOptions>();
var dbContext = services.GetRequiredService <PortalDbContext>();
var spider = await dbContext.Spiders.FirstOrDefaultAsync(x => x.Id == int.Parse(jobId));
if (spider == null)
{
logger.LogError($"任务 {jobId} 不存在");
return;
}
if (!spider.Enable)
{
logger.LogError($"任务 {jobId} 被禁用");
return;
}
var client = new DockerClientConfiguration(
new Uri(options.Docker))
.CreateClient();
var batch = Guid.NewGuid().ToString("N");
var env = new List <string>((spider.Environment ?? "").Split(new[] { " " },
StringSplitOptions.RemoveEmptyEntries))
{
$"DOTNET_SPIDER_ID={batch}",
$"DOTNET_SPIDER_TYPE={spider.Type}",
$"DOTNET_SPIDER_NAME={spider.Name}"
};
var image = string.IsNullOrWhiteSpace(spider.Registry)
? $"{spider.Repository}:{spider.Tag}"
: $"{spider.Registry}/{spider.Repository}:{spider.Tag}";
var name = $"dotnetspider-{spider.Id}-{batch}";
var parameters = new CreateContainerParameters
{
Image = image,
Name = name,
Labels = new Dictionary <string, string>
{
{ "dotnetspider.spider.id", spider.Id.ToString() },
{ "dotnetspider.spider.batch", batch },
{ "dotnetspider.spider.type", spider.Type },
{ "dotnetspider.spider.name", spider.Name }
},
Env = env,
HostConfig = new HostConfig()
};
parameters.HostConfig.Binds = options.DockerVolumes;
var result = await client.Containers.CreateContainerAsync(parameters);
if (result.ID == null)
{
logger.LogError($"创建任务 {jobId} 实例失败: {string.Join(", ", result.Warnings)}");
}
var spiderContainer = new SpiderContainer
{
ContainerId = result.ID,
Batch = batch,
SpiderId = spider.Id,
Status = "Created",
CreationTime = DateTimeOffset.Now
};
dbContext.SpiderContainers.Add(spiderContainer);
await dbContext.SaveChangesAsync();
var startResult =
await client.Containers.StartContainerAsync(result.ID, new ContainerStartParameters());
spiderContainer.Status = startResult ? "Success" : "Failed";
await dbContext.SaveChangesAsync();
logger.LogInformation($"触发任务 {jobId} 完成");
}
catch (Exception ex)
{
logger.LogError($"触发任务 {jobId} 失败: {ex}");
}
}
}
