public async Task InitData()
{
var dataSource = _project.DataSource;
dataSource.Paramters.EnsureValue("DbProvider", out string dbProvider);
dataSource.Paramters.EnsureValue("ConnectionString", out string connString);
dataSource.Paramters.Value("PKColumn", out string pkColumn);
dataSource.Paramters.Value("ModifyTime", out string modifyTime);
dataSource.Paramters.EnsureValue("Query", out string queryCmd);
#region CreateSqlMapper
var smartSqlOptions = new CreateSmartSqlMapperOptions
{
LoggerFactory = _loggerFactory,
ProviderName = dbProvider,
Alias = Name,
DataSource = new SmartSql.Configuration.WriteDataSource
{
Name = Name,
ConnectionString = connString
}
};
var sqlMapper = Create(smartSqlOptions);
#endregion
var lastExtract = await _etlRepository.GetLastExtract(_project.GetETLCode());
_project.SetETLLastExtract(lastExtract);
var queryParams = new Dictionary <string, object>
{
{ "LastMaxId", lastExtract.MaxId },
{ "LastQueryTime", lastExtract.QueryTime },
{ "LastMaxModifyTime", lastExtract.MaxModifyTime },
};
var extractEntity = new ETLExtract
{
QueryTime = DateTime.Now,
PKColumn = pkColumn,
QueryCommand = new ETLDbCommand
{
Command = queryCmd,
Paramters = queryParams
},
QuerySize = -1,
MaxId = -1
};
Stopwatch stopwatch = Stopwatch.StartNew();
DbSet = await sqlMapper.GetDbSetAsync(new RequestContext { RealSql = queryCmd, Request = queryParams });
TransformData = DbSet.Tables.First();
stopwatch.Stop();
extractEntity.QuerySize = TransformData.Rows.Count;
extractEntity.QueryCommand.Taken = stopwatch.ElapsedMilliseconds;
_logger.LogWarning($"InitData,Data.Size:{extractEntity.QuerySize},Taken:{extractEntity.QueryCommand.Taken}ms!");
dataSource.Paramters.Value("PkIsNumeric", out bool pkIsNumeric);
dataSource.Paramters.Value("AutoIncrement", out bool autoIncrement);
if (!String.IsNullOrEmpty(pkColumn) &&
(pkIsNumeric || autoIncrement) &&
extractEntity.QuerySize > 0)
{
var maxId = TransformData.Rows.Max(m => m.Cells[pkColumn].Value);
extractEntity.MaxId = Convert.ToInt64(maxId);
}
else
{
extractEntity.MaxId = lastExtract.MaxId;
}
if (!String.IsNullOrEmpty(modifyTime) &&
extractEntity.QuerySize > 0)
{
var maxModifyTime = TransformData.Rows.Max(m => m.Cells[modifyTime].Value);
extractEntity.MaxModifyTime = Convert.ToDateTime(maxModifyTime);
}
else
{
extractEntity.MaxModifyTime = lastExtract.MaxModifyTime;
}
await _etlRepository.Extract(_project.GetETKTaskId(), extractEntity);
}
public async Task InitData()
{
var dataSource = _project.DataSource;
dataSource.Parameters.EnsureValue("DbProvider", out string dbProvider);
dataSource.Parameters.EnsureValue("ConnectionString", out string connString);
dataSource.Parameters.Value("PKColumn", out string pkColumn);
dataSource.Parameters.Value("ModifyTime", out string modifyTime);
dataSource.Parameters.EnsureValue("Query", out string queryCmd);
#region CreateSqlMapper
var sqlMapper = new SmartSqlBuilder()
.UseAlias(Name)
.UseLoggerFactory(_loggerFactory)
.UseDataSource(dbProvider, connString)
.Build().SqlMapper;
#endregion
var lastExtract = await _etlRepository.GetLastExtract(_project.GetETLCode());
_project.SetETLLastExtract(lastExtract);
var queryParams = new Dictionary <string, object>
{
{ "LastMaxId", lastExtract.MaxId },
{ "LastQueryTime", lastExtract.QueryTime },
{ "LastMaxModifyTime", lastExtract.MaxModifyTime },
};
var extractEntity = new ETLExtract
{
QueryTime = DateTime.Now,
PKColumn = pkColumn,
QueryCommand = new ETLDbCommand
{
Command = queryCmd,
Parameters = queryParams
},
QuerySize = -1,
MaxId = -1
};
Stopwatch stopwatch = Stopwatch.StartNew();
DataSet = await sqlMapper.GetDataSetAsync(new RequestContext { RealSql = queryCmd, Request = queryParams });
TransformData = DataSet.Tables[0];
stopwatch.Stop();
extractEntity.QuerySize = TransformData.Rows.Count;
extractEntity.QueryCommand.Taken = stopwatch.ElapsedMilliseconds;
_logger.LogWarning($"InitData,Data.Size:{extractEntity.QuerySize},Taken:{extractEntity.QueryCommand.Taken}ms!");
dataSource.Parameters.Value("PkIsNumeric", out bool pkIsNumeric);
dataSource.Parameters.Value("AutoIncrement", out bool autoIncrement);
if (!String.IsNullOrEmpty(pkColumn) &&
(pkIsNumeric || autoIncrement) &&
extractEntity.QuerySize > 0)
{
var maxId = TransformData.Rows.Cast <DataRow>().AsParallel().Max(dr => dr[pkColumn]);
extractEntity.MaxId = Convert.ToInt64(maxId);
}
else
{
extractEntity.MaxId = lastExtract.MaxId;
}
if (!String.IsNullOrEmpty(modifyTime) &&
extractEntity.QuerySize > 0)
{
extractEntity.MaxModifyTime = TransformData.Rows.Cast <DataRow>().AsParallel().Max(dr =>
{
var cell = dr[modifyTime];
return(Convert.ToDateTime(cell));
});
}
else
{
extractEntity.MaxModifyTime = lastExtract.MaxModifyTime;
}
await _etlRepository.Extract(_project.GetETKTaskId(), extractEntity);
}
public async Task Run()
{
InitParameters();
LastExtract = await _etlRepository.GetLastExtract(_project.GetETLCode());
_project.SetETLLastExtract(LastExtract);
var queryParams = new Dictionary <string, object>
{
{ "LastMaxId", LastExtract.MaxId },
{ "LastQueryTime", LastExtract.QueryTime },
{ "LastMaxModifyTime", LastExtract.MaxModifyTime }
};
Total = await SqlMapper.ExecuteScalarAsync <int>(new RequestContext
{
RealSql = TotalCmd,
Request = queryParams
});
if (Total == 0)
{
_logger.LogInformation("can not find any record.");
return;
}
if (BulkSize == 0)
{
BulkSize = Total;
}
BuildContext buildContext = null;
while (Offset < Total)
{
_logger.LogInformation($"--------Total:[{Total}] , Offset:[{Offset}] ---------");
try
{
var etlTaskId = await _etlRepository.Startup(_project.ConfigPath, _project.GetETLCode());
_project.SetETKTaskId(etlTaskId);
await Extract();
if (String.IsNullOrEmpty(TotalCmd))
{
Total = GetCount();
BulkSize = Total;
}
foreach (var buildKV in _project.BuildTasks)
{
_logger.LogInformation($"-------- BuildTask:{buildKV.Key} Start! ---------");
var output = buildKV.Value.Output;
buildContext = new BuildContext
{
PluginManager = _pluginManager,
Project = _project,
BuildKey = buildKV.Key,
Build = buildKV.Value,
Output = output?.Copy()
};
buildContext.SetExtractData(this);
await _pluginManager.Resolve <IBuildTask>(buildKV.Value.Type).Build(buildContext);
_logger.LogInformation($"-------- BuildTask:{buildKV.Key} End! ---------");
}
await _etlRepository.Success(_project.GetETKTaskId());
Offset += BulkSize;
}
catch (Exception e)
{
await _etlRepository.Fail(_project.GetETKTaskId(), e);
throw;
}
}
}
