private JobLog GetJobLog(TableName tableName, DataPumperContext dataPumperContext) { var tableSync = dataPumperContext.TableSyncs.FirstOrDefault(ts => ts.TableName == tableName.SourceFullName); var jobLog = dataPumperContext.Logs.OrderByDescending(l => l.Id).FirstOrDefault(l => l.TableSyncId == tableSync.Id); return(jobLog); }
private void UpdateJobLog(object sender, ProgressEventArgs args) { using (var logContext = new DataPumperContext(_configuration.MetadataConnectionString)) { var logRecord = GetJobLog(args.TableName, logContext); logRecord.RecordsProcessed = args.Processed; logContext.SaveChanges(); } }
public DataPumpService(IEnumerable <IDataPumperSource> sources, IEnumerable <IDataPumperTarget> targets, DataPumper.Core.DataPumper pumper, DataPumperContext context, ILogger <DataPumpService> logger) { _pumper = pumper; _context = context; _logger = logger; Sources = sources; Targets = targets; }
public Task <List <JobLog> > GetLogRecords(int skip, int take) { using (var ctx = new DataPumperContext(_configuration.MetadataConnectionString)) { return(ctx.Logs .OrderByDescending(r => r.StartDate) .AsNoTracking() .Skip(skip) .Take(take) .ToListAsync()); } }
public WarehouseService() { _configSource = new ConfigurationBuilder() .AddXmlFile("data-pumper.config") .AddXmlFile("data-pumper.local.config", true) .Build(); _configuration = new WarehouseServiceConfiguration(_configSource); ConfigurationManager.Configuration = _configSource; using (var ctx = new DataPumperContext(_configuration.MetadataConnectionString)) { ctx.TableSyncs.ToList(); } }
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env, DataPumperContext context) { if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); } else { app.UseExceptionHandler("/Error"); // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts. app.UseHsts(); } app.UseHttpsRedirection(); app.UseStaticFiles(); app.UseRouting(); app.UseHangfireDashboard("/jobs"); app.UseEndpoints(endpoints => { endpoints.MapControllerRoute( name: "default", pattern: "{controller=Home}/{action=Index}/{id?}"); endpoints.MapBlazorHub(); endpoints.MapFallbackToPage("/_Host"); }); context.Database.Migrate(); context.Seed(); var cron = context.Settings.FirstOrDefault(s => s.Key == Setting.Cron)?.Value ?? "0 30 3 ? * *"; RecurringJob.AddOrUpdate <DataPumpService>(MainService.JobId, s => s.Process(false), cron); RecurringJob.AddOrUpdate <DataPumpService>(MainService.FullReloadJobId, s => s.Process(true), Cron.Never); }
private async Task <JobLog> RunJobInternal(PumperJobItem job, IDataPumperSource sourceProvider, IDataPumperTarget targetProvider, bool fullReloading) { Log.Warn($"Processing {job.Name}"); using (var ctx = new DataPumperContext(_configuration.MetadataConnectionString)) { var tableSync = await ctx.TableSyncs.FirstOrDefaultAsync(ts => ts.TableName == job.TargetTableName); if (tableSync == null) { tableSync = new TableSync { TableName = job.TargetTableName }; ctx.TableSyncs.Add(tableSync); } var jobLog = new JobLog { TableSync = tableSync }; ctx.Logs.Add(jobLog); await ctx.SaveChangesAsync(); try { var sw = new Stopwatch(); sw.Start(); targetProvider.Progress += UpdateJobLog; await targetProvider.RunQuery(job.PreRunQuery); if (fullReloading) { // При полной перезаливке обнуляем ActualDate tableSync.ActualDate = null; } var jobActualDate = tableSync.ActualDate; // Если переливка не выполнялась, то будет Null var onDate = jobActualDate ?? DateTime.Today.AddYears(-100); var currentDate = await sourceProvider.GetCurrentDate(_configuration.CurrentDateQuery) ?? DateTime.Now.Date; if (job.HistoricMode) { if (currentDate == tableSync.ActualDate) { onDate = tableSync.PreviousActualDate ?? DateTime.Today.AddYears(-100); } else { tableSync.PreviousActualDate = tableSync.ActualDate; } } var records = await _pumper.Pump(sourceProvider, targetProvider, new PumpParameters( new TableName(job.SourceTableName), new TableName(job.TargetTableName), _configuration.ActualityColumnName, _configuration.HistoricColumnFrom, _configuration.TenantField, onDate.AddDays(_configuration.BackwardReloadDays), job.HistoricMode, currentDate, fullReloading, _tenantCodes) { DeleteProtectionDate = _configuration.DeleteProtectionDate }); tableSync.ActualDate = currentDate; jobLog.EndDate = DateTime.Now; jobLog.RecordsProcessed = records; jobLog.Status = SyncStatus.Success; await targetProvider.RunQuery(job.PostRunQuery); sw.Stop(); } catch (Exception ex) { Log.Error($"Error processing job {job}", ex); jobLog.Message = ex.Message; jobLog.Status = SyncStatus.Error; } await ctx.SaveChangesAsync(); return(jobLog); } }