public static async Task InitializeAsync(PortalOptions options, IServiceProvider serviceProvider)
{
using (var scope = serviceProvider.GetRequiredService <IServiceScopeFactory>()
.CreateScope())
{
var context = scope.ServiceProvider.GetRequiredService <PortalDbContext>();
await context.Database.MigrateAsync();
await context.Database.EnsureCreatedAsync();
string sql;
var conn = context.Database.GetDbConnection();
Stream sqlStream = null;
switch (options.DatabaseType?.ToLower())
{
case "mysql":
{
if (await conn.QuerySingleAsync <int>(
$"SELECT count(*) FROM information_schema.TABLES WHERE table_name ='QRTZ_FIRED_TRIGGERS';") ==
0
)
{
sqlStream = typeof(SeedData).Assembly
.GetManifestResourceStream("DotnetSpider.Portal.DDL.MySql.sql");
}
break;
}
default:
{
if (await conn.QuerySingleAsync <int>(
$"SELECT COUNT(*) from sysobjects WHERE id = object_id(N'[dbo].[QRTZ_FIRED_TRIGGERS]') AND OBJECTPROPERTY(id, N'') = IsUserTable") ==
0
)
{
sqlStream = typeof(SeedData).Assembly
.GetManifestResourceStream("DotnetSpider.Portal.DDL.SqlServer.sql");
}
break;
}
}
if (sqlStream == null)
{
throw new SpiderException("Can't find quartz.net MySql sql");
}
using (var reader = new StreamReader(sqlStream))
{
sql = await reader.ReadToEndAsync();
await conn.ExecuteAsync(sql);
}
var sched = serviceProvider.GetRequiredService <IScheduler>();
await InitializeSeedDataAsync(context, sched);
}
}
public static async Task RunAsync(PortalOptions options, PortalDbContext dbContext, int jobId)
{
var spider = await dbContext.Spiders.FirstOrDefaultAsync(x => x.Id == jobId);
if (spider == null)
{
throw new Exception($"任务 {jobId} 不存在");
}
DockerClient client = new DockerClientConfiguration(
new Uri("http://localhost:2376"))
.CreateClient();
var env = new List <string>((spider.Environment ?? "").Split(new[] { " " },
StringSplitOptions.RemoveEmptyEntries))
{
$"id={spider.Id}",
$"type={spider.Type}",
$"name={spider.Name}"
};
var image = $"{spider.Registry}{spider.Repository}:{spider.Tag}";
var result = await client.Containers.CreateContainerAsync(new CreateContainerParameters
{
Image = image,
Name = $"dotnetspider-{spider.Id}",
Labels = new Dictionary <string, string>
{
{ "dotnetspider.spider.id", spider.Id.ToString() },
{ "dotnetspider.spider.type", spider.Type },
{ "dotnetspider.spider.name", spider.Name }
},
Volumes = new Dictionary <string, EmptyStruct>
{
{
options.DockerVolumes, new EmptyStruct()
}
},
Env = env
});
if (result.ID == null)
{
throw new Exception($"创建任务 {jobId} 实例失败: {string.Join(", ", result.Warnings)}");
}
var spiderContainer = new SpiderContainer
{
ContainerId = result.ID,
SpiderId = spider.Id,
Status = "OK"
};
dbContext.SpiderContainers.Add(spiderContainer);
await dbContext.SaveChangesAsync();
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.TryAddSingleton <PortalOptions>();
services.AddControllersWithViews()
.SetCompatibilityVersion(CompatibilityVersion.Version_3_0)
.AddNewtonsoftJson()
.AddRazorRuntimeCompilation();
services.AddHealthChecks();
var options = new PortalOptions(Configuration);
// Add DbContext
Action <DbContextOptionsBuilder> dbContextOptionsBuilder;
var migrationsAssembly = typeof(Startup).GetTypeInfo().Assembly.GetName().Name;
switch (options.DatabaseType?.ToLower())
{
case "mysql":
{
dbContextOptionsBuilder = b =>
b.UseMySql(options.ConnectionString,
sql => sql.MigrationsAssembly(migrationsAssembly));
break;
}
default:
{
dbContextOptionsBuilder = b =>
b.UseSqlServer(options.ConnectionString,
sql => sql.MigrationsAssembly(migrationsAssembly));
break;
}
}
services.AddDbContext <PortalDbContext>(dbContextOptionsBuilder);
services.AddQuartz(x =>
{
x.UseMySqlConnector(options.ConnectionString);
});
services.AddHttpClient();
services.Configure <AgentCenterOptions>(Configuration);
services.AddHttpClient();
services.AddAgentCenter <MySqlAgentStore>();
services.AddStatistics <MySqlStatisticsStore>();
services.AddRabbitMQ(Configuration.GetSection("RabbitMQ"));
services.AddHostedService <QuartzService>();
services.AddHostedService <CleanDockerContainerService>();
services.AddSingleton <IActionResultTypeMapper, ActionResultTypeMapper>();
services.AddRouting(x =>
{
x.LowercaseUrls = true;
});
services.AddAutoMapper(typeof(AutoMapperProfile));
}
public PortalDbContext CreateDbContext(string[] args)
{
var builder = new DbContextOptionsBuilder <PortalDbContext>();
var configuration = Framework.CreateConfiguration(args.Length > 0 ? args[0] : "appsettings.json");
var options = new PortalOptions(configuration);
switch (options.Database?.ToLower())
{
case "mysql":
{
builder.UseMySql(options.ConnectionString);
break;
}
default:
{
builder.UseSqlServer(options.ConnectionString);
break;
}
}
return(new PortalDbContext(builder.Options, true));
}
public CleanDockerContainerService(PortalOptions options, ILogger <CleanDockerContainerService> logger)
{
_options = options;
_logger = logger;
}
public Startup(IConfiguration configuration)
{
Configuration = configuration;
_options = new PortalOptions(Configuration);
}
public static async Task RunAsync(PortalOptions options, PortalDbContext dbContext, int jobId)
{
var spider = await dbContext.Spiders.FirstOrDefaultAsync(x => x.Id == jobId);
if (spider == null)
{
throw new Exception($"任务 {jobId} 不存在");
}
var docker = new DockerClient.DockerClient(new Uri(options.Docker));
bool exists = await docker.ExistsAsync(new
{
label = new[] { $"dotnetspider.spider.id={spider.Id}" }
});
if (exists)
{
throw new Exception($"任务 {spider.Id} 正在运行");
}
var env = new List <string>((spider.Environment ?? "").Split(new[] { " " },
StringSplitOptions.RemoveEmptyEntries))
{
$"id={spider.Id}",
$"type={spider.Type}",
$"name={spider.Name}"
};
var result = await docker.PullAsync(spider.Image);
if (!result.Success)
{
throw new Exception($"接取镜像 {spider.Image} 失败: {result.Message}");
}
result = await docker.CreateAsync(spider.Image,
env.ToArray(),
new Dictionary <string, object>
{
{ "id", spider.Id.ToString() },
{ "type", spider.Type },
{ "name", spider.Name }
}, new[] { options.DockerVolumes });
if (!result.Success)
{
throw new Exception($"创建任务 {jobId} 实例失败: {result.Message}");
}
var spiderContainer = new SpiderContainer
{
ContainerId = result.Id,
SpiderId = spider.Id,
Status = "Creating"
};
result = await docker.StartAsync(spiderContainer.ContainerId);
if (result.Success)
{
spiderContainer.Status = "OK";
}
dbContext.SpiderContainers.Add(spiderContainer);
await dbContext.SaveChangesAsync();
}
