private static async Task InitializeSeedDataAsync(PortalDbContext context, IScheduler sched)
{
if (!await context.Spiders.AnyAsync())
{
var spider = new Data.Spider
{
Name = "cnblogs",
Cron = "0 1 */1 * * ?",
Type = "DotnetSpider.Spiders.CnblogsSpider",
Image = "dotnetspider/spiders.startup:latest",
CreationTime = DateTimeOffset.Now,
Enabled = true,
LastModificationTime = DateTimeOffset.Now
};
await context.Spiders.AddAsync(spider);
await context.SaveChangesAsync();
var trigger = TriggerBuilder.Create().WithCronSchedule(spider.Cron)
.WithIdentity(spider.Id.ToString())
.Build();
var qzJob = JobBuilder.Create <QuartzJob>().WithIdentity(spider.Id.ToString())
.WithDescription(spider.Name)
.RequestRecovery(true).Build();
await sched.ScheduleJob(qzJob, trigger);
}
}
public static async Task RunAsync(PortalOptions options, PortalDbContext dbContext, int jobId)
{
var spider = await dbContext.Spiders.FirstOrDefaultAsync(x => x.Id == jobId);
if (spider == null)
{
throw new Exception($"任务 {jobId} 不存在");
}
DockerClient client = new DockerClientConfiguration(
new Uri("http://localhost:2376"))
.CreateClient();
var env = new List <string>((spider.Environment ?? "").Split(new[] { " " },
StringSplitOptions.RemoveEmptyEntries))
{
$"id={spider.Id}",
$"type={spider.Type}",
$"name={spider.Name}"
};
var image = $"{spider.Registry}{spider.Repository}:{spider.Tag}";
var result = await client.Containers.CreateContainerAsync(new CreateContainerParameters
{
Image = image,
Name = $"dotnetspider-{spider.Id}",
Labels = new Dictionary <string, string>
{
{ "dotnetspider.spider.id", spider.Id.ToString() },
{ "dotnetspider.spider.type", spider.Type },
{ "dotnetspider.spider.name", spider.Name }
},
Volumes = new Dictionary <string, EmptyStruct>
{
{
options.DockerVolumes, new EmptyStruct()
}
},
Env = env
});
if (result.ID == null)
{
throw new Exception($"创建任务 {jobId} 实例失败: {string.Join(", ", result.Warnings)}");
}
var spiderContainer = new SpiderContainer
{
ContainerId = result.ID,
SpiderId = spider.Id,
Status = "OK"
};
dbContext.SpiderContainers.Add(spiderContainer);
await dbContext.SaveChangesAsync();
}
private static async Task InitializeSeedDataAsync(PortalDbContext context, IScheduler sched)
{
if (!await context.DockerRepositories.AnyAsync())
{
var repo = new DockerRepository
{
Name = "DockerHub",
Schema = null,
Registry = null,
Repository = "dotnetspider/spiders.startup",
CreationTime = DateTimeOffset.Now,
UserName = "",
Password = ""
};
await context.DockerRepositories.AddAsync(repo);
var spider = new Data.Spider
{
Name = "cnblogs",
Cron = "0 1 */1 * * ?",
Repository = "dotnetspider/spiders.startup",
Type = "DotnetSpider.Spiders.CnblogsSpider",
Tag = "latest",
CreationTime = DateTimeOffset.Now
};
await context.Spiders.AddAsync(spider);
await context.SaveChangesAsync();
var trigger = TriggerBuilder.Create().WithCronSchedule(spider.Cron)
.WithIdentity(spider.Id.ToString())
.Build();
var qzJob = JobBuilder.Create <QuartzJob>().WithIdentity(spider.Id.ToString())
.WithDescription(spider.Name)
.RequestRecovery(true).Build();
await sched.ScheduleJob(qzJob, trigger);
}
}
public static async Task RunAsync(PortalOptions options, PortalDbContext dbContext, int jobId)
{
var spider = await dbContext.Spiders.FirstOrDefaultAsync(x => x.Id == jobId);
if (spider == null)
{
throw new Exception($"任务 {jobId} 不存在");
}
var docker = new DockerClient.DockerClient(new Uri(options.Docker));
bool exists = await docker.ExistsAsync(new
{
label = new[] { $"dotnetspider.spider.id={spider.Id}" }
});
if (exists)
{
throw new Exception($"任务 {spider.Id} 正在运行");
}
var env = new List <string>((spider.Environment ?? "").Split(new[] { " " },
StringSplitOptions.RemoveEmptyEntries))
{
$"id={spider.Id}",
$"type={spider.Type}",
$"name={spider.Name}"
};
var result = await docker.PullAsync(spider.Image);
if (!result.Success)
{
throw new Exception($"接取镜像 {spider.Image} 失败: {result.Message}");
}
result = await docker.CreateAsync(spider.Image,
env.ToArray(),
new Dictionary <string, object>
{
{ "id", spider.Id.ToString() },
{ "type", spider.Type },
{ "name", spider.Name }
}, new[] { options.DockerVolumes });
if (!result.Success)
{
throw new Exception($"创建任务 {jobId} 实例失败: {result.Message}");
}
var spiderContainer = new SpiderContainer
{
ContainerId = result.Id,
SpiderId = spider.Id,
Status = "Creating"
};
result = await docker.StartAsync(spiderContainer.ContainerId);
if (result.Success)
{
spiderContainer.Status = "OK";
}
dbContext.SpiderContainers.Add(spiderContainer);
await dbContext.SaveChangesAsync();
}
