public async Task RunJobs(IDataPumperSource sourceProvider, IDataPumperTarget targetProvider, bool fullReloading = false)
{
Log.Info("Running all jobs...");
var jobs = _configuration.Jobs;
var logs = await ProcessInternal(jobs, sourceProvider, targetProvider, fullReloading);
LogsSender.Send(logs.ToList());
}
public async Task <long> Pump(IDataPumperSource source, IDataPumperTarget target, PumpParameters parameters)
{
try
{
var sw = new Stopwatch();
sw.Start();
if (parameters.HistoricMode)
{
Log.Info($"Cleaning target table in history mode '{parameters.TargetTable}' (historyDateFrom {parameters.CurrentDate}) for instances: {string.Join(",", parameters.TenantCodes ?? new string[0])}...");
await target.CleanupHistoryTable(new CleanupTableRequest(
parameters.TargetTable,
parameters.HistoryDateFromFieldName,
parameters.TenantField,
parameters.TenantCodes,
parameters.CurrentDate,
parameters.FullReloading)
{
DeleteProtectionDate = parameters.DeleteProtectionDate
});
Log.Info($"Cleaning '{parameters.TargetTable}' complete in {sw.Elapsed}, transferring data...");
sw.Restart();
}
else
{
Log.Info($"Cleaning target table '{parameters.TargetTable}' (from date {parameters.OnDate}) for instances: {string.Join(",", parameters.TenantCodes ?? new string[0])}...");
await target.CleanupTable(new CleanupTableRequest(
parameters.TargetTable,
parameters.ActualityFieldName,
parameters.OnDate,
parameters.TenantField,
parameters.TenantCodes,
parameters.FullReloading)
{
DeleteProtectionDate = parameters.DeleteProtectionDate
});
Log.Info($"Cleaning '{parameters.TargetTable}' complete in {sw.Elapsed}, transferring data...");
sw.Restart();
}
using (var reader = await source.GetDataReader(parameters.SourceTable, parameters.ActualityFieldName, parameters.OnDate, parameters.TenantField, parameters.TenantCodes))
{
var items = await target.InsertData(parameters.TargetTable, reader);
Log.Info($"Data transfer '{parameters.TargetTable}' of {items} records completed in {sw.Elapsed}");
return(items);
}
}
catch (Exception ex)
{
Log.Error($"Error processing {parameters.SourceTable} -> {parameters.TargetTable}", ex);
throw;
}
}
private async Task <JobLog> RunJobInternal(PumperJobItem job, IDataPumperSource sourceProvider, IDataPumperTarget targetProvider, bool fullReloading)
{
Log.Warn($"Processing {job.Name}");
using (var ctx = new DataPumperContext(_configuration.MetadataConnectionString))
{
var tableSync = await ctx.TableSyncs.FirstOrDefaultAsync(ts => ts.TableName == job.TargetTableName);
if (tableSync == null)
{
tableSync = new TableSync
{
TableName = job.TargetTableName
};
ctx.TableSyncs.Add(tableSync);
}
var jobLog = new JobLog {
TableSync = tableSync
};
ctx.Logs.Add(jobLog);
await ctx.SaveChangesAsync();
try
{
var sw = new Stopwatch();
sw.Start();
targetProvider.Progress += UpdateJobLog;
await targetProvider.RunQuery(job.PreRunQuery);
if (fullReloading)
{
// При полной перезаливке обнуляем ActualDate
tableSync.ActualDate = null;
}
var jobActualDate = tableSync.ActualDate; // Если переливка не выполнялась, то будет Null
var onDate = jobActualDate ?? DateTime.Today.AddYears(-100);
var currentDate = await sourceProvider.GetCurrentDate(_configuration.CurrentDateQuery) ?? DateTime.Now.Date;
if (job.HistoricMode)
{
if (currentDate == tableSync.ActualDate)
{
onDate = tableSync.PreviousActualDate ?? DateTime.Today.AddYears(-100);
}
else
{
tableSync.PreviousActualDate = tableSync.ActualDate;
}
}
var records = await _pumper.Pump(sourceProvider, targetProvider,
new PumpParameters(
new TableName(job.SourceTableName),
new TableName(job.TargetTableName),
_configuration.ActualityColumnName,
_configuration.HistoricColumnFrom,
_configuration.TenantField,
onDate.AddDays(_configuration.BackwardReloadDays),
job.HistoricMode,
currentDate,
fullReloading,
_tenantCodes)
{
DeleteProtectionDate = _configuration.DeleteProtectionDate
});
tableSync.ActualDate = currentDate;
jobLog.EndDate = DateTime.Now;
jobLog.RecordsProcessed = records;
jobLog.Status = SyncStatus.Success;
await targetProvider.RunQuery(job.PostRunQuery);
sw.Stop();
}
catch (Exception ex)
{
Log.Error($"Error processing job {job}", ex);
jobLog.Message = ex.Message;
jobLog.Status = SyncStatus.Error;
}
await ctx.SaveChangesAsync();
return(jobLog);
}
}
public async Task <IEnumerable <JobLog> > ProcessInternal(PumperJobItem[] jobs, IDataPumperSource sourceProvider, IDataPumperTarget targetProvider,
bool fullReloading)
{
Log.Warn("Started job to sync all tables...");
var jobLogs = new List <JobLog>();
foreach (var job in jobs)
{
var jobLog = await RunJobInternal(job, sourceProvider, targetProvider, fullReloading);
jobLogs.Add(jobLog);
}
return(jobLogs);
}
public async Task RunJob(PumperJobItem jobItem, IDataPumperSource sourceProvider, IDataPumperTarget targetProvider, bool fullReloading = false)
{
Log.Info($"Performing synchronization for job '{jobItem.Name}'... ");
var log = await RunJobInternal(jobItem, sourceProvider, targetProvider, fullReloading);
LogsSender.Send(new[] { log });
}
