private void InitColumnMapping(IBatchInsert batchInsert, BuildContext context)
{
if (context.Build.Paramters.Value(COLUMN_MAPPING, out IEnumerable colMapps))
{
foreach (IDictionary <object, object> colMappingKV in colMapps)
{
colMappingKV.EnsureValue("Column", out string colName);
colMappingKV.EnsureValue("Mapping", out string mapping);
colMappingKV.Value("DataTypeName", out string dataTypeName);
var colMapping = new ColumnMapping
{
Column = colName,
Mapping = mapping,
DataTypeName = dataTypeName
};
batchInsert.AddColumnMapping(colMapping);
}
}
}
public async Task Build(BuildContext context)
{
context.Build.Paramters.EnsureValue(TABLE_NAME, out string tableName);
context.Build.Paramters.EnsureValue(DB_PROVIDER, out DbProvider dbProvider);
var etlRepository = _pluginManager.Resolve <IETLRepository>(_project.GetETLRepository());
var dataSource = context.GetDataSource <ExtractDataSource>();
if (dataSource.TransformData.Rows.Count == 0)
{
await etlRepository.Load(_project.GetETKTaskId(), new Entity.ETLLoad
{
Size = 0,
Table = tableName
});
return;
}
var batchTable = dataSource.TransformData;
batchTable.Name = tableName;
var sqlMapper = GetSqlMapper(context);
context.Build.Paramters.Value(PRE_COMMAND, out string preCmd);
var lastExtract = _project.GetETLLastExtract();
var queryParams = new Dictionary <string, object>
{
{ "LastMaxId", lastExtract.MaxId },
{ "LastQueryTime", lastExtract.QueryTime },
{ "LastMaxModifyTime", lastExtract.MaxModifyTime },
};
Stopwatch stopwatch = Stopwatch.StartNew();
var loadEntity = new Entity.ETLLoad
{
Table = tableName
};
if (!String.IsNullOrEmpty(preCmd))
{
stopwatch.Restart();
await sqlMapper.ExecuteAsync(new RequestContext
{
RealSql = preCmd,
Request = queryParams
});
stopwatch.Stop();
loadEntity.PreCommand = new Entity.ETLDbCommand
{
Command = preCmd,
Paramters = queryParams,
Taken = stopwatch.ElapsedMilliseconds
};
}
using (IBatchInsert batchInsert = BatchInsertFactory.Create(sqlMapper, dbProvider))
{
InitColumnMapping(batchInsert, context);
batchInsert.Table = batchTable;
stopwatch.Restart();
await batchInsert.InsertAsync();
stopwatch.Stop();
loadEntity.Size = batchTable.Rows.Count;
loadEntity.Taken = stopwatch.ElapsedMilliseconds;
_logger.LogWarning($"Build:{context.BuildKey},BatchInsert.Size:{loadEntity.Size},Taken:{loadEntity.Taken}ms!");
}
if (context.Build.Paramters.Value(POST_COMMAND, out string postCmd) && !String.IsNullOrEmpty(postCmd))
{
stopwatch.Restart();
await sqlMapper.ExecuteAsync(new RequestContext
{
RealSql = postCmd,
Request = queryParams
});
stopwatch.Stop();
loadEntity.PostCommand = new Entity.ETLDbCommand
{
Command = postCmd,
Paramters = queryParams,
Taken = stopwatch.ElapsedMilliseconds
};
}
await etlRepository.Load(_project.GetETKTaskId(), loadEntity);
}
