public async Task InitData() { var dataSource = _project.DataSource; dataSource.Paramters.EnsureValue("DbProvider", out string dbProvider); dataSource.Paramters.EnsureValue("ConnectionString", out string connString); dataSource.Paramters.Value("PKColumn", out string pkColumn); dataSource.Paramters.Value("ModifyTime", out string modifyTime); dataSource.Paramters.EnsureValue("Query", out string queryCmd); #region CreateSqlMapper var smartSqlOptions = new CreateSmartSqlMapperOptions { LoggerFactory = _loggerFactory, ProviderName = dbProvider, Alias = Name, DataSource = new SmartSql.Configuration.WriteDataSource { Name = Name, ConnectionString = connString } }; var sqlMapper = Create(smartSqlOptions); #endregion var lastExtract = await _etlRepository.GetLastExtract(_project.GetETLCode()); _project.SetETLLastExtract(lastExtract); var queryParams = new Dictionary <string, object> { { "LastMaxId", lastExtract.MaxId }, { "LastQueryTime", lastExtract.QueryTime }, { "LastMaxModifyTime", lastExtract.MaxModifyTime }, }; var extractEntity = new ETLExtract { QueryTime = DateTime.Now, PKColumn = pkColumn, QueryCommand = new ETLDbCommand { Command = queryCmd, Paramters = queryParams }, QuerySize = -1, MaxId = -1 }; Stopwatch stopwatch = Stopwatch.StartNew(); DbSet = await sqlMapper.GetDbSetAsync(new RequestContext { RealSql = queryCmd, Request = queryParams }); TransformData = DbSet.Tables.First(); stopwatch.Stop(); extractEntity.QuerySize = TransformData.Rows.Count; extractEntity.QueryCommand.Taken = stopwatch.ElapsedMilliseconds; _logger.LogWarning($"InitData,Data.Size:{extractEntity.QuerySize},Taken:{extractEntity.QueryCommand.Taken}ms!"); dataSource.Paramters.Value("PkIsNumeric", out bool pkIsNumeric); dataSource.Paramters.Value("AutoIncrement", out bool autoIncrement); if (!String.IsNullOrEmpty(pkColumn) && (pkIsNumeric || autoIncrement) && extractEntity.QuerySize > 0) { var maxId = TransformData.Rows.Max(m => m.Cells[pkColumn].Value); extractEntity.MaxId = Convert.ToInt64(maxId); } else { extractEntity.MaxId = lastExtract.MaxId; } if (!String.IsNullOrEmpty(modifyTime) && extractEntity.QuerySize > 0) { var maxModifyTime = TransformData.Rows.Max(m => m.Cells[modifyTime].Value); extractEntity.MaxModifyTime = Convert.ToDateTime(maxModifyTime); } else { extractEntity.MaxModifyTime = lastExtract.MaxModifyTime; } await _etlRepository.Extract(_project.GetETKTaskId(), extractEntity); }
public async Task InitData() { var dataSource = _project.DataSource; dataSource.Parameters.EnsureValue("DbProvider", out string dbProvider); dataSource.Parameters.EnsureValue("ConnectionString", out string connString); dataSource.Parameters.Value("PKColumn", out string pkColumn); dataSource.Parameters.Value("ModifyTime", out string modifyTime); dataSource.Parameters.EnsureValue("Query", out string queryCmd); #region CreateSqlMapper var sqlMapper = new SmartSqlBuilder() .UseAlias(Name) .UseLoggerFactory(_loggerFactory) .UseDataSource(dbProvider, connString) .Build().SqlMapper; #endregion var lastExtract = await _etlRepository.GetLastExtract(_project.GetETLCode()); _project.SetETLLastExtract(lastExtract); var queryParams = new Dictionary <string, object> { { "LastMaxId", lastExtract.MaxId }, { "LastQueryTime", lastExtract.QueryTime }, { "LastMaxModifyTime", lastExtract.MaxModifyTime }, }; var extractEntity = new ETLExtract { QueryTime = DateTime.Now, PKColumn = pkColumn, QueryCommand = new ETLDbCommand { Command = queryCmd, Parameters = queryParams }, QuerySize = -1, MaxId = -1 }; Stopwatch stopwatch = Stopwatch.StartNew(); DataSet = await sqlMapper.GetDataSetAsync(new RequestContext { RealSql = queryCmd, Request = queryParams }); TransformData = DataSet.Tables[0]; stopwatch.Stop(); extractEntity.QuerySize = TransformData.Rows.Count; extractEntity.QueryCommand.Taken = stopwatch.ElapsedMilliseconds; _logger.LogWarning($"InitData,Data.Size:{extractEntity.QuerySize},Taken:{extractEntity.QueryCommand.Taken}ms!"); dataSource.Parameters.Value("PkIsNumeric", out bool pkIsNumeric); dataSource.Parameters.Value("AutoIncrement", out bool autoIncrement); if (!String.IsNullOrEmpty(pkColumn) && (pkIsNumeric || autoIncrement) && extractEntity.QuerySize > 0) { var maxId = TransformData.Rows.Cast <DataRow>().AsParallel().Max(dr => dr[pkColumn]); extractEntity.MaxId = Convert.ToInt64(maxId); } else { extractEntity.MaxId = lastExtract.MaxId; } if (!String.IsNullOrEmpty(modifyTime) && extractEntity.QuerySize > 0) { extractEntity.MaxModifyTime = TransformData.Rows.Cast <DataRow>().AsParallel().Max(dr => { var cell = dr[modifyTime]; return(Convert.ToDateTime(cell)); }); } else { extractEntity.MaxModifyTime = lastExtract.MaxModifyTime; } await _etlRepository.Extract(_project.GetETKTaskId(), extractEntity); }
public async Task Run() { InitParameters(); LastExtract = await _etlRepository.GetLastExtract(_project.GetETLCode()); _project.SetETLLastExtract(LastExtract); var queryParams = new Dictionary <string, object> { { "LastMaxId", LastExtract.MaxId }, { "LastQueryTime", LastExtract.QueryTime }, { "LastMaxModifyTime", LastExtract.MaxModifyTime } }; Total = await SqlMapper.ExecuteScalarAsync <int>(new RequestContext { RealSql = TotalCmd, Request = queryParams }); if (Total == 0) { _logger.LogInformation("can not find any record."); return; } if (BulkSize == 0) { BulkSize = Total; } BuildContext buildContext = null; while (Offset < Total) { _logger.LogInformation($"--------Total:[{Total}] , Offset:[{Offset}] ---------"); try { var etlTaskId = await _etlRepository.Startup(_project.ConfigPath, _project.GetETLCode()); _project.SetETKTaskId(etlTaskId); await Extract(); if (String.IsNullOrEmpty(TotalCmd)) { Total = GetCount(); BulkSize = Total; } foreach (var buildKV in _project.BuildTasks) { _logger.LogInformation($"-------- BuildTask:{buildKV.Key} Start! ---------"); var output = buildKV.Value.Output; buildContext = new BuildContext { PluginManager = _pluginManager, Project = _project, BuildKey = buildKV.Key, Build = buildKV.Value, Output = output?.Copy() }; buildContext.SetExtractData(this); await _pluginManager.Resolve <IBuildTask>(buildKV.Value.Type).Build(buildContext); _logger.LogInformation($"-------- BuildTask:{buildKV.Key} End! ---------"); } await _etlRepository.Success(_project.GetETKTaskId()); Offset += BulkSize; } catch (Exception e) { await _etlRepository.Fail(_project.GetETKTaskId(), e); throw; } } }