private async Task Extract()
{
var queryParams = new Dictionary <string, object>
{
{ "LastMaxId", LastExtract.MaxId },
{ "LastQueryTime", LastExtract.QueryTime },
{ "LastMaxModifyTime", LastExtract.MaxModifyTime },
{ nameof(Offset), Offset },
{ nameof(BulkSize), BulkSize },
};
var extractEntity = new ETLExtract
{
QueryTime = DateTime.Now,
PKColumn = PKColumn,
QueryCommand = new ETLDbCommand
{
Command = QueryCmd,
Parameters = queryParams
},
Count = -1,
MaxId = -1
};
Stopwatch stopwatch = Stopwatch.StartNew();
await LoadData(new RequestContext { RealSql = QueryCmd, Request = queryParams });
stopwatch.Stop();
extractEntity.Count = GetCount();
extractEntity.QueryCommand.Taken = stopwatch.ElapsedMilliseconds;
_logger.LogWarning($"InitData,Data.Size:{extractEntity.Count},Taken:{extractEntity.QueryCommand.Taken}ms!");
if (!String.IsNullOrEmpty(PKColumn) &&
(PkIsNumeric || AutoIncrement) &&
extractEntity.Count > 0)
{
extractEntity.MaxId = GetMaxId(PKColumn);
}
else
{
extractEntity.MaxId = LastExtract.MaxId;
}
if (!String.IsNullOrEmpty(ModifyTime) &&
extractEntity.Count > 0)
{
extractEntity.MaxModifyTime = GetMaxModifyTime(ModifyTime);
}
else
{
extractEntity.MaxModifyTime = LastExtract.MaxModifyTime;
}
await _etlRepository.Extract(_project.GetETKTaskId(), extractEntity);
}
public async Task InitData()
{
var dataSource = _project.DataSource;
dataSource.Parameters.EnsureValue("DbProvider", out string dbProvider);
dataSource.Parameters.EnsureValue("ConnectionString", out string connString);
dataSource.Parameters.Value("PKColumn", out string pkColumn);
dataSource.Parameters.Value("ModifyTime", out string modifyTime);
dataSource.Parameters.EnsureValue("Query", out string queryCmd);
#region CreateSqlMapper
var sqlMapper = new SmartSqlBuilder()
.UseAlias(Name)
.UseLoggerFactory(_loggerFactory)
.UseDataSource(dbProvider, connString)
.Build().SqlMapper;
#endregion
var lastExtract = await _etlRepository.GetLastExtract(_project.GetETLCode());
_project.SetETLLastExtract(lastExtract);
var queryParams = new Dictionary <string, object>
{
{ "LastMaxId", lastExtract.MaxId },
{ "LastQueryTime", lastExtract.QueryTime },
{ "LastMaxModifyTime", lastExtract.MaxModifyTime },
};
var extractEntity = new ETLExtract
{
QueryTime = DateTime.Now,
PKColumn = pkColumn,
QueryCommand = new ETLDbCommand
{
Command = queryCmd,
Parameters = queryParams
},
QuerySize = -1,
MaxId = -1
};
Stopwatch stopwatch = Stopwatch.StartNew();
DataSet = await sqlMapper.GetDataSetAsync(new RequestContext { RealSql = queryCmd, Request = queryParams });
TransformData = DataSet.Tables[0];
stopwatch.Stop();
extractEntity.QuerySize = TransformData.Rows.Count;
extractEntity.QueryCommand.Taken = stopwatch.ElapsedMilliseconds;
_logger.LogWarning($"InitData,Data.Size:{extractEntity.QuerySize},Taken:{extractEntity.QueryCommand.Taken}ms!");
dataSource.Parameters.Value("PkIsNumeric", out bool pkIsNumeric);
dataSource.Parameters.Value("AutoIncrement", out bool autoIncrement);
if (!String.IsNullOrEmpty(pkColumn) &&
(pkIsNumeric || autoIncrement) &&
extractEntity.QuerySize > 0)
{
var maxId = TransformData.Rows.Cast <DataRow>().AsParallel().Max(dr => dr[pkColumn]);
extractEntity.MaxId = Convert.ToInt64(maxId);
}
else
{
extractEntity.MaxId = lastExtract.MaxId;
}
if (!String.IsNullOrEmpty(modifyTime) &&
extractEntity.QuerySize > 0)
{
extractEntity.MaxModifyTime = TransformData.Rows.Cast <DataRow>().AsParallel().Max(dr =>
{
var cell = dr[modifyTime];
return(Convert.ToDateTime(cell));
});
}
else
{
extractEntity.MaxModifyTime = lastExtract.MaxModifyTime;
}
await _etlRepository.Extract(_project.GetETKTaskId(), extractEntity);
}
public async Task InitData()
{
var dataSource = _project.DataSource;
dataSource.Paramters.EnsureValue("DbProvider", out string dbProvider);
dataSource.Paramters.EnsureValue("ConnectionString", out string connString);
dataSource.Paramters.Value("PKColumn", out string pkColumn);
dataSource.Paramters.Value("ModifyTime", out string modifyTime);
dataSource.Paramters.EnsureValue("Query", out string queryCmd);
#region CreateSqlMapper
var smartSqlOptions = new CreateSmartSqlMapperOptions
{
LoggerFactory = _loggerFactory,
ProviderName = dbProvider,
Alias = Name,
DataSource = new SmartSql.Configuration.WriteDataSource
{
Name = Name,
ConnectionString = connString
}
};
var sqlMapper = Create(smartSqlOptions);
#endregion
var lastExtract = await _etlRepository.GetLastExtract(_project.GetETLCode());
_project.SetETLLastExtract(lastExtract);
var queryParams = new Dictionary <string, object>
{
{ "LastMaxId", lastExtract.MaxId },
{ "LastQueryTime", lastExtract.QueryTime },
{ "LastMaxModifyTime", lastExtract.MaxModifyTime },
};
var extractEntity = new ETLExtract
{
QueryTime = DateTime.Now,
PKColumn = pkColumn,
QueryCommand = new ETLDbCommand
{
Command = queryCmd,
Paramters = queryParams
},
QuerySize = -1,
MaxId = -1
};
Stopwatch stopwatch = Stopwatch.StartNew();
DbSet = await sqlMapper.GetDbSetAsync(new RequestContext { RealSql = queryCmd, Request = queryParams });
TransformData = DbSet.Tables.First();
stopwatch.Stop();
extractEntity.QuerySize = TransformData.Rows.Count;
extractEntity.QueryCommand.Taken = stopwatch.ElapsedMilliseconds;
_logger.LogWarning($"InitData,Data.Size:{extractEntity.QuerySize},Taken:{extractEntity.QueryCommand.Taken}ms!");
dataSource.Paramters.Value("PkIsNumeric", out bool pkIsNumeric);
dataSource.Paramters.Value("AutoIncrement", out bool autoIncrement);
if (!String.IsNullOrEmpty(pkColumn) &&
(pkIsNumeric || autoIncrement) &&
extractEntity.QuerySize > 0)
{
var maxId = TransformData.Rows.Max(m => m.Cells[pkColumn].Value);
extractEntity.MaxId = Convert.ToInt64(maxId);
}
else
{
extractEntity.MaxId = lastExtract.MaxId;
}
if (!String.IsNullOrEmpty(modifyTime) &&
extractEntity.QuerySize > 0)
{
var maxModifyTime = TransformData.Rows.Max(m => m.Cells[modifyTime].Value);
extractEntity.MaxModifyTime = Convert.ToDateTime(maxModifyTime);
}
else
{
extractEntity.MaxModifyTime = lastExtract.MaxModifyTime;
}
await _etlRepository.Extract(_project.GetETKTaskId(), extractEntity);
}
