public async Task RunJob(PumperJobItem jobItem, IDataPumperSource sourceProvider, IDataPumperTarget targetProvider, bool fullReloading = false) { Log.Info($"Performing synchronization for job '{jobItem.Name}'... "); var log = await RunJobInternal(jobItem, sourceProvider, targetProvider, fullReloading); LogsSender.Send(new[] { log }); }
public async Task RunJobs(IDataPumperSource sourceProvider, IDataPumperTarget targetProvider, bool fullReloading = false) { Log.Info("Running all jobs..."); var jobs = _configuration.Jobs; var logs = await ProcessInternal(jobs, sourceProvider, targetProvider, fullReloading); LogsSender.Send(logs.ToList()); }
public async Task <long> Pump(IDataPumperSource source, IDataPumperTarget target, PumpParameters parameters) { try { var sw = new Stopwatch(); sw.Start(); if (parameters.HistoricMode) { Log.Info($"Cleaning target table in history mode '{parameters.TargetTable}' (historyDateFrom {parameters.CurrentDate}) for instances: {string.Join(",", parameters.TenantCodes ?? new string[0])}..."); await target.CleanupHistoryTable(new CleanupTableRequest( parameters.TargetTable, parameters.HistoryDateFromFieldName, parameters.TenantField, parameters.TenantCodes, parameters.CurrentDate, parameters.FullReloading) { DeleteProtectionDate = parameters.DeleteProtectionDate }); Log.Info($"Cleaning '{parameters.TargetTable}' complete in {sw.Elapsed}, transferring data..."); sw.Restart(); } else { Log.Info($"Cleaning target table '{parameters.TargetTable}' (from date {parameters.OnDate}) for instances: {string.Join(",", parameters.TenantCodes ?? new string[0])}..."); await target.CleanupTable(new CleanupTableRequest( parameters.TargetTable, parameters.ActualityFieldName, parameters.OnDate, parameters.TenantField, parameters.TenantCodes, parameters.FullReloading) { DeleteProtectionDate = parameters.DeleteProtectionDate }); Log.Info($"Cleaning '{parameters.TargetTable}' complete in {sw.Elapsed}, transferring data..."); sw.Restart(); } using (var reader = await source.GetDataReader(parameters.SourceTable, parameters.ActualityFieldName, parameters.OnDate, parameters.TenantField, parameters.TenantCodes)) { var items = await target.InsertData(parameters.TargetTable, reader); Log.Info($"Data transfer '{parameters.TargetTable}' of {items} records completed in {sw.Elapsed}"); return(items); } } catch (Exception ex) { Log.Error($"Error processing {parameters.SourceTable} -> {parameters.TargetTable}", ex); throw; } }
private async Task <DateTime?> GetCurrentDate(IDataPumperSource source) { var curDateTable = (await _context.Settings.FirstOrDefaultAsync(s => s.Key == Setting.CurrentDateTable))?.Value; var curDateField = (await _context.Settings.FirstOrDefaultAsync(s => s.Key == Setting.CurrentDateField))?.Value; if (string.IsNullOrEmpty(curDateTable) || string.IsNullOrEmpty(curDateField)) { return(null); } return(await source.GetCurrentDate(new TableName(curDateTable), curDateField)); }
private async Task <JobLog> RunJobInternal(PumperJobItem job, IDataPumperSource sourceProvider, IDataPumperTarget targetProvider, bool fullReloading) { Log.Warn($"Processing {job.Name}"); using (var ctx = new DataPumperContext(_configuration.MetadataConnectionString)) { var tableSync = await ctx.TableSyncs.FirstOrDefaultAsync(ts => ts.TableName == job.TargetTableName); if (tableSync == null) { tableSync = new TableSync { TableName = job.TargetTableName }; ctx.TableSyncs.Add(tableSync); } var jobLog = new JobLog { TableSync = tableSync }; ctx.Logs.Add(jobLog); await ctx.SaveChangesAsync(); try { var sw = new Stopwatch(); sw.Start(); targetProvider.Progress += UpdateJobLog; await targetProvider.RunQuery(job.PreRunQuery); if (fullReloading) { // При полной перезаливке обнуляем ActualDate tableSync.ActualDate = null; } var jobActualDate = tableSync.ActualDate; // Если переливка не выполнялась, то будет Null var onDate = jobActualDate ?? DateTime.Today.AddYears(-100); var currentDate = await sourceProvider.GetCurrentDate(_configuration.CurrentDateQuery) ?? DateTime.Now.Date; if (job.HistoricMode) { if (currentDate == tableSync.ActualDate) { onDate = tableSync.PreviousActualDate ?? DateTime.Today.AddYears(-100); } else { tableSync.PreviousActualDate = tableSync.ActualDate; } } var records = await _pumper.Pump(sourceProvider, targetProvider, new PumpParameters( new TableName(job.SourceTableName), new TableName(job.TargetTableName), _configuration.ActualityColumnName, _configuration.HistoricColumnFrom, _configuration.TenantField, onDate.AddDays(_configuration.BackwardReloadDays), job.HistoricMode, currentDate, fullReloading, _tenantCodes) { DeleteProtectionDate = _configuration.DeleteProtectionDate }); tableSync.ActualDate = currentDate; jobLog.EndDate = DateTime.Now; jobLog.RecordsProcessed = records; jobLog.Status = SyncStatus.Success; await targetProvider.RunQuery(job.PostRunQuery); sw.Stop(); } catch (Exception ex) { Log.Error($"Error processing job {job}", ex); jobLog.Message = ex.Message; jobLog.Status = SyncStatus.Error; } await ctx.SaveChangesAsync(); return(jobLog); } }
public async Task <IEnumerable <JobLog> > ProcessInternal(PumperJobItem[] jobs, IDataPumperSource sourceProvider, IDataPumperTarget targetProvider, bool fullReloading) { Log.Warn("Started job to sync all tables..."); var jobLogs = new List <JobLog>(); foreach (var job in jobs) { var jobLog = await RunJobInternal(job, sourceProvider, targetProvider, fullReloading); jobLogs.Add(jobLog); } return(jobLogs); }