public async Task Build(BuildContext context) { ESOptions esOptions = InitOptions(context); var etlRepository = _pluginManager.Resolve <IETLTaskRepository>(_project.GetETLRepository()); var dataSource = context.GetDataSource <ExtractDataSource>(); var loadEntity = new Entity.ETLLoad { Size = 0, Paramters = new Dictionary <String, object> { { "Task", "LoadToES" }, { "Index", esOptions.Index }, { "Type", esOptions.TypeName } } }; if (dataSource.TransformData.Rows.Count == 0) { await etlRepository.Load(_project.GetETKTaskId(), loadEntity); return; } Stopwatch stopwatch = Stopwatch.StartNew(); #region InitColumnMapping var colMappings = InitColumnMapping(context); if (colMappings != null) { foreach (var colMapping in colMappings) { if (dataSource.TransformData.Columns.ContainsKey(colMapping.Column)) { dataSource.TransformData.Columns[colMapping.Column].Name = colMapping.Mapping; } } } #endregion #region BatchInsert var esClient = GetElasticClient(esOptions); var list = new List <Dictionary <String, object> >(); foreach (var row in dataSource.TransformData.Rows) { var item = new Dictionary <String, object>(); foreach (var cellKV in row.Cells) { var cellVal = cellKV.Value; item.Add(cellVal.Column.Name, cellVal.Value); } list.Add(item); } var indexExResp = await esClient.IndexExistsAsync(esOptions.Index); if (!indexExResp.Exists) { var createIndexResp = await esClient.CreateIndexAsync(esOptions.Index); } var esSyncResp = await esClient.BulkAsync((bulkRequest) => { var bulkReqDesc = bulkRequest .Index(esOptions.Index) .Type(esOptions.TypeName); if (context.Build.Paramters.Value(ID_MAPPING, out string es_id)) { return(bulkReqDesc.IndexMany(list, (bulkIdxDesc, item) => { var idVal = item[es_id].ToString(); return bulkIdxDesc.Id(idVal); } )); } return(bulkReqDesc.IndexMany(list)); } ); if (esSyncResp.Errors || !esSyncResp.IsValid) { _logger.LogError($"ES.ERRORS:{esSyncResp.DebugInformation}"); throw new SmartCodeException($"ES.ERRORS:{esSyncResp.DebugInformation}"); } stopwatch.Stop(); loadEntity.Size = dataSource.TransformData.Rows.Count; loadEntity.Taken = stopwatch.ElapsedMilliseconds; #endregion await etlRepository.Load(_project.GetETKTaskId(), loadEntity); }
public async Task Build(BuildContext context) { context.Build.Paramters.EnsureValue(TABLE_NAME, out string tableName); context.Build.Paramters.EnsureValue(DB_PROVIDER, out DbProvider dbProvider); var etlRepository = _pluginManager.Resolve <IETLRepository>(_project.GetETLRepository()); var dataSource = context.GetDataSource <ExtractDataSource>(); if (dataSource.TransformData.Rows.Count == 0) { await etlRepository.Load(_project.GetETKTaskId(), new Entity.ETLLoad { Size = 0, Table = tableName }); return; } var batchTable = dataSource.TransformData; batchTable.Name = tableName; var sqlMapper = GetSqlMapper(context); context.Build.Paramters.Value(PRE_COMMAND, out string preCmd); var lastExtract = _project.GetETLLastExtract(); var queryParams = new Dictionary <string, object> { { "LastMaxId", lastExtract.MaxId }, { "LastQueryTime", lastExtract.QueryTime }, { "LastMaxModifyTime", lastExtract.MaxModifyTime }, }; Stopwatch stopwatch = Stopwatch.StartNew(); var loadEntity = new Entity.ETLLoad { Table = tableName }; if (!String.IsNullOrEmpty(preCmd)) { stopwatch.Restart(); await sqlMapper.ExecuteAsync(new RequestContext { RealSql = preCmd, Request = queryParams }); stopwatch.Stop(); loadEntity.PreCommand = new Entity.ETLDbCommand { Command = preCmd, Paramters = queryParams, Taken = stopwatch.ElapsedMilliseconds }; } using (IBatchInsert batchInsert = BatchInsertFactory.Create(sqlMapper, dbProvider)) { InitColumnMapping(batchInsert, context); batchInsert.Table = batchTable; stopwatch.Restart(); await batchInsert.InsertAsync(); stopwatch.Stop(); loadEntity.Size = batchTable.Rows.Count; loadEntity.Taken = stopwatch.ElapsedMilliseconds; _logger.LogWarning($"Build:{context.BuildKey},BatchInsert.Size:{loadEntity.Size},Taken:{loadEntity.Taken}ms!"); } if (context.Build.Paramters.Value(POST_COMMAND, out string postCmd) && !String.IsNullOrEmpty(postCmd)) { stopwatch.Restart(); await sqlMapper.ExecuteAsync(new RequestContext { RealSql = postCmd, Request = queryParams }); stopwatch.Stop(); loadEntity.PostCommand = new Entity.ETLDbCommand { Command = postCmd, Paramters = queryParams, Taken = stopwatch.ElapsedMilliseconds }; } await etlRepository.Load(_project.GetETKTaskId(), loadEntity); }
public async Task Build(BuildContext context) { ESOptions esOptions = InitOptions(context); var etlRepository = _pluginManager.Resolve <IETLTaskRepository>(_project.GetETLRepository()); var dataSource = context.GetExtractData <ExtractDictionaryData>(); var loadEntity = new Entity.ETLLoad { Size = 0, Parameters = new Dictionary <String, object> { { "Task", "LoadToES" }, { "Index", esOptions.Index }, { "Type", esOptions.TypeName } } }; if (dataSource.TransformData.Count == 0) { await etlRepository.Load(_project.GetETKTaskId(), loadEntity); return; } Stopwatch stopwatch = Stopwatch.StartNew(); #region InitColumnMapping var colMapping = new Dictionary <string, string>(); if (context.Build.Parameters.Value(COLUMN_MAPPING, out IEnumerable colMapps)) { foreach (IDictionary <object, object> colMappingKV in colMapps) { colMappingKV.EnsureValue("Column", out string colName); colMappingKV.EnsureValue("Mapping", out string mapping); colMapping.Add(colName, mapping); } } dataSource.TransformData = dataSource.TransformData.Select((dic) => { IDictionary <string, object> newItem = new Dictionary <string, object>(); foreach (KeyValuePair <string, object> item in dic) { var itemKey = item.Key; if (colMapps != null) { if (colMapping.TryGetValue(itemKey, out string mapping)) { itemKey = mapping; } } newItem.Add(itemKey, item.Value); } return(newItem); }).ToList(); #endregion #region BatchInsert var esClient = GetElasticClient(esOptions); var indexExResp = await esClient.IndexExistsAsync(esOptions.Index); if (!indexExResp.Exists) { var createIndexResp = await esClient.CreateIndexAsync(esOptions.Index); } var esSyncResp = await esClient.BulkAsync((bulkRequest) => { var bulkReqDesc = bulkRequest .Index(esOptions.Index) .Type(esOptions.TypeName); if (context.Build.Parameters.Value(ID_MAPPING, out string es_id)) { return(bulkReqDesc.IndexMany(dataSource.TransformData, (bulkIdxDesc, item) => { var idVal = item[es_id].ToString(); return bulkIdxDesc.Id(idVal); } )); } return(bulkReqDesc.IndexMany(dataSource.TransformData)); } ); if (esSyncResp.Errors || !esSyncResp.IsValid) { _logger.LogError($"ES.ERRORS:{esSyncResp.DebugInformation}"); throw new SmartCodeException($"ES.ERRORS:{esSyncResp.DebugInformation}"); } stopwatch.Stop(); loadEntity.Size = dataSource.TransformData.Count; loadEntity.Taken = stopwatch.ElapsedMilliseconds; #endregion await etlRepository.Load(_project.GetETKTaskId(), loadEntity); }