public static async Task InitializeAsync(PortalOptions options, IServiceProvider serviceProvider) { using (var scope = serviceProvider.GetRequiredService <IServiceScopeFactory>() .CreateScope()) { var context = scope.ServiceProvider.GetRequiredService <PortalDbContext>(); await context.Database.MigrateAsync(); await context.Database.EnsureCreatedAsync(); string sql; var conn = context.Database.GetDbConnection(); Stream sqlStream = null; switch (options.DatabaseType?.ToLower()) { case "mysql": { if (await conn.QuerySingleAsync <int>( $"SELECT count(*) FROM information_schema.TABLES WHERE table_name ='QRTZ_FIRED_TRIGGERS';") == 0 ) { sqlStream = typeof(SeedData).Assembly .GetManifestResourceStream("DotnetSpider.Portal.DDL.MySql.sql"); } break; } default: { if (await conn.QuerySingleAsync <int>( $"SELECT COUNT(*) from sysobjects WHERE id = object_id(N'[dbo].[QRTZ_FIRED_TRIGGERS]') AND OBJECTPROPERTY(id, N'') = IsUserTable") == 0 ) { sqlStream = typeof(SeedData).Assembly .GetManifestResourceStream("DotnetSpider.Portal.DDL.SqlServer.sql"); } break; } } if (sqlStream == null) { throw new SpiderException("Can't find quartz.net MySql sql"); } using (var reader = new StreamReader(sqlStream)) { sql = await reader.ReadToEndAsync(); await conn.ExecuteAsync(sql); } var sched = serviceProvider.GetRequiredService <IScheduler>(); await InitializeSeedDataAsync(context, sched); } }
public static async Task RunAsync(PortalOptions options, PortalDbContext dbContext, int jobId) { var spider = await dbContext.Spiders.FirstOrDefaultAsync(x => x.Id == jobId); if (spider == null) { throw new Exception($"任务 {jobId} 不存在"); } DockerClient client = new DockerClientConfiguration( new Uri("http://localhost:2376")) .CreateClient(); var env = new List <string>((spider.Environment ?? "").Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries)) { $"id={spider.Id}", $"type={spider.Type}", $"name={spider.Name}" }; var image = $"{spider.Registry}{spider.Repository}:{spider.Tag}"; var result = await client.Containers.CreateContainerAsync(new CreateContainerParameters { Image = image, Name = $"dotnetspider-{spider.Id}", Labels = new Dictionary <string, string> { { "dotnetspider.spider.id", spider.Id.ToString() }, { "dotnetspider.spider.type", spider.Type }, { "dotnetspider.spider.name", spider.Name } }, Volumes = new Dictionary <string, EmptyStruct> { { options.DockerVolumes, new EmptyStruct() } }, Env = env }); if (result.ID == null) { throw new Exception($"创建任务 {jobId} 实例失败: {string.Join(", ", result.Warnings)}"); } var spiderContainer = new SpiderContainer { ContainerId = result.ID, SpiderId = spider.Id, Status = "OK" }; dbContext.SpiderContainers.Add(spiderContainer); await dbContext.SaveChangesAsync(); }
// This method gets called by the runtime. Use this method to add services to the container. public void ConfigureServices(IServiceCollection services) { services.TryAddSingleton <PortalOptions>(); services.AddControllersWithViews() .SetCompatibilityVersion(CompatibilityVersion.Version_3_0) .AddNewtonsoftJson() .AddRazorRuntimeCompilation(); services.AddHealthChecks(); var options = new PortalOptions(Configuration); // Add DbContext Action <DbContextOptionsBuilder> dbContextOptionsBuilder; var migrationsAssembly = typeof(Startup).GetTypeInfo().Assembly.GetName().Name; switch (options.DatabaseType?.ToLower()) { case "mysql": { dbContextOptionsBuilder = b => b.UseMySql(options.ConnectionString, sql => sql.MigrationsAssembly(migrationsAssembly)); break; } default: { dbContextOptionsBuilder = b => b.UseSqlServer(options.ConnectionString, sql => sql.MigrationsAssembly(migrationsAssembly)); break; } } services.AddDbContext <PortalDbContext>(dbContextOptionsBuilder); services.AddQuartz(x => { x.UseMySqlConnector(options.ConnectionString); }); services.AddHttpClient(); services.Configure <AgentCenterOptions>(Configuration); services.AddHttpClient(); services.AddAgentCenter <MySqlAgentStore>(); services.AddStatistics <MySqlStatisticsStore>(); services.AddRabbitMQ(Configuration.GetSection("RabbitMQ")); services.AddHostedService <QuartzService>(); services.AddHostedService <CleanDockerContainerService>(); services.AddSingleton <IActionResultTypeMapper, ActionResultTypeMapper>(); services.AddRouting(x => { x.LowercaseUrls = true; }); services.AddAutoMapper(typeof(AutoMapperProfile)); }
public PortalDbContext CreateDbContext(string[] args) { var builder = new DbContextOptionsBuilder <PortalDbContext>(); var configuration = Framework.CreateConfiguration(args.Length > 0 ? args[0] : "appsettings.json"); var options = new PortalOptions(configuration); switch (options.Database?.ToLower()) { case "mysql": { builder.UseMySql(options.ConnectionString); break; } default: { builder.UseSqlServer(options.ConnectionString); break; } } return(new PortalDbContext(builder.Options, true)); }
public CleanDockerContainerService(PortalOptions options, ILogger <CleanDockerContainerService> logger) { _options = options; _logger = logger; }
public Startup(IConfiguration configuration) { Configuration = configuration; _options = new PortalOptions(Configuration); }
public static async Task RunAsync(PortalOptions options, PortalDbContext dbContext, int jobId) { var spider = await dbContext.Spiders.FirstOrDefaultAsync(x => x.Id == jobId); if (spider == null) { throw new Exception($"任务 {jobId} 不存在"); } var docker = new DockerClient.DockerClient(new Uri(options.Docker)); bool exists = await docker.ExistsAsync(new { label = new[] { $"dotnetspider.spider.id={spider.Id}" } }); if (exists) { throw new Exception($"任务 {spider.Id} 正在运行"); } var env = new List <string>((spider.Environment ?? "").Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries)) { $"id={spider.Id}", $"type={spider.Type}", $"name={spider.Name}" }; var result = await docker.PullAsync(spider.Image); if (!result.Success) { throw new Exception($"接取镜像 {spider.Image} 失败: {result.Message}"); } result = await docker.CreateAsync(spider.Image, env.ToArray(), new Dictionary <string, object> { { "id", spider.Id.ToString() }, { "type", spider.Type }, { "name", spider.Name } }, new[] { options.DockerVolumes }); if (!result.Success) { throw new Exception($"创建任务 {jobId} 实例失败: {result.Message}"); } var spiderContainer = new SpiderContainer { ContainerId = result.Id, SpiderId = spider.Id, Status = "Creating" }; result = await docker.StartAsync(spiderContainer.ContainerId); if (result.Success) { spiderContainer.Status = "OK"; } dbContext.SpiderContainers.Add(spiderContainer); await dbContext.SaveChangesAsync(); }