private JobLog GetJobLog(TableName tableName, DataPumperContext dataPumperContext)
{
var tableSync = dataPumperContext.TableSyncs.FirstOrDefault(ts => ts.TableName == tableName.SourceFullName);
var jobLog = dataPumperContext.Logs.OrderByDescending(l => l.Id).FirstOrDefault(l => l.TableSyncId == tableSync.Id);
return(jobLog);
}
private void UpdateJobLog(object sender, ProgressEventArgs args)
{
using (var logContext = new DataPumperContext(_configuration.MetadataConnectionString))
{
var logRecord = GetJobLog(args.TableName, logContext);
logRecord.RecordsProcessed = args.Processed;
logContext.SaveChanges();
}
}
public DataPumpService(IEnumerable <IDataPumperSource> sources,
IEnumerable <IDataPumperTarget> targets,
DataPumper.Core.DataPumper pumper,
DataPumperContext context,
ILogger <DataPumpService> logger)
{
_pumper = pumper;
_context = context;
_logger = logger;
Sources = sources;
Targets = targets;
}
public Task <List <JobLog> > GetLogRecords(int skip, int take)
{
using (var ctx = new DataPumperContext(_configuration.MetadataConnectionString))
{
return(ctx.Logs
.OrderByDescending(r => r.StartDate)
.AsNoTracking()
.Skip(skip)
.Take(take)
.ToListAsync());
}
}
public WarehouseService()
{
_configSource = new ConfigurationBuilder()
.AddXmlFile("data-pumper.config")
.AddXmlFile("data-pumper.local.config", true)
.Build();
_configuration = new WarehouseServiceConfiguration(_configSource);
ConfigurationManager.Configuration = _configSource;
using (var ctx = new DataPumperContext(_configuration.MetadataConnectionString))
{
ctx.TableSyncs.ToList();
}
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, DataPumperContext context)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
else
{
app.UseExceptionHandler("/Error");
// The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
app.UseHsts();
}
app.UseHttpsRedirection();
app.UseStaticFiles();
app.UseRouting();
app.UseHangfireDashboard("/jobs");
app.UseEndpoints(endpoints =>
{
endpoints.MapControllerRoute(
name: "default",
pattern: "{controller=Home}/{action=Index}/{id?}");
endpoints.MapBlazorHub();
endpoints.MapFallbackToPage("/_Host");
});
context.Database.Migrate();
context.Seed();
var cron = context.Settings.FirstOrDefault(s => s.Key == Setting.Cron)?.Value ?? "0 30 3 ? * *";
RecurringJob.AddOrUpdate <DataPumpService>(MainService.JobId, s => s.Process(false), cron);
RecurringJob.AddOrUpdate <DataPumpService>(MainService.FullReloadJobId, s => s.Process(true), Cron.Never);
}
private async Task <JobLog> RunJobInternal(PumperJobItem job, IDataPumperSource sourceProvider, IDataPumperTarget targetProvider, bool fullReloading)
{
Log.Warn($"Processing {job.Name}");
using (var ctx = new DataPumperContext(_configuration.MetadataConnectionString))
{
var tableSync = await ctx.TableSyncs.FirstOrDefaultAsync(ts => ts.TableName == job.TargetTableName);
if (tableSync == null)
{
tableSync = new TableSync
{
TableName = job.TargetTableName
};
ctx.TableSyncs.Add(tableSync);
}
var jobLog = new JobLog {
TableSync = tableSync
};
ctx.Logs.Add(jobLog);
await ctx.SaveChangesAsync();
try
{
var sw = new Stopwatch();
sw.Start();
targetProvider.Progress += UpdateJobLog;
await targetProvider.RunQuery(job.PreRunQuery);
if (fullReloading)
{
// При полной перезаливке обнуляем ActualDate
tableSync.ActualDate = null;
}
var jobActualDate = tableSync.ActualDate; // Если переливка не выполнялась, то будет Null
var onDate = jobActualDate ?? DateTime.Today.AddYears(-100);
var currentDate = await sourceProvider.GetCurrentDate(_configuration.CurrentDateQuery) ?? DateTime.Now.Date;
if (job.HistoricMode)
{
if (currentDate == tableSync.ActualDate)
{
onDate = tableSync.PreviousActualDate ?? DateTime.Today.AddYears(-100);
}
else
{
tableSync.PreviousActualDate = tableSync.ActualDate;
}
}
var records = await _pumper.Pump(sourceProvider, targetProvider,
new PumpParameters(
new TableName(job.SourceTableName),
new TableName(job.TargetTableName),
_configuration.ActualityColumnName,
_configuration.HistoricColumnFrom,
_configuration.TenantField,
onDate.AddDays(_configuration.BackwardReloadDays),
job.HistoricMode,
currentDate,
fullReloading,
_tenantCodes)
{
DeleteProtectionDate = _configuration.DeleteProtectionDate
});
tableSync.ActualDate = currentDate;
jobLog.EndDate = DateTime.Now;
jobLog.RecordsProcessed = records;
jobLog.Status = SyncStatus.Success;
await targetProvider.RunQuery(job.PostRunQuery);
sw.Stop();
}
catch (Exception ex)
{
Log.Error($"Error processing job {job}", ex);
jobLog.Message = ex.Message;
jobLog.Status = SyncStatus.Error;
}
await ctx.SaveChangesAsync();
return(jobLog);
}
}
