Exemple #1
0
 private void InitColumnMapping(IBatchInsert batchInsert, BuildContext context)
 {
     if (context.Build.Paramters.Value(COLUMN_MAPPING, out IEnumerable colMapps))
     {
         foreach (IDictionary <object, object> colMappingKV in colMapps)
         {
             colMappingKV.EnsureValue("Column", out string colName);
             colMappingKV.EnsureValue("Mapping", out string mapping);
             colMappingKV.Value("DataTypeName", out string dataTypeName);
             var colMapping = new ColumnMapping
             {
                 Column       = colName,
                 Mapping      = mapping,
                 DataTypeName = dataTypeName
             };
             batchInsert.AddColumnMapping(colMapping);
         }
     }
 }
Exemple #2
0
        public async Task Build(BuildContext context)
        {
            context.Build.Paramters.EnsureValue(TABLE_NAME, out string tableName);
            context.Build.Paramters.EnsureValue(DB_PROVIDER, out DbProvider dbProvider);
            var etlRepository = _pluginManager.Resolve <IETLRepository>(_project.GetETLRepository());
            var dataSource    = context.GetDataSource <ExtractDataSource>();

            if (dataSource.TransformData.Rows.Count == 0)
            {
                await etlRepository.Load(_project.GetETKTaskId(), new Entity.ETLLoad
                {
                    Size  = 0,
                    Table = tableName
                });

                return;
            }
            var batchTable = dataSource.TransformData;

            batchTable.Name = tableName;
            var sqlMapper = GetSqlMapper(context);

            context.Build.Paramters.Value(PRE_COMMAND, out string preCmd);
            var lastExtract = _project.GetETLLastExtract();
            var queryParams = new Dictionary <string, object>
            {
                { "LastMaxId", lastExtract.MaxId },
                { "LastQueryTime", lastExtract.QueryTime },
                { "LastMaxModifyTime", lastExtract.MaxModifyTime },
            };
            Stopwatch stopwatch  = Stopwatch.StartNew();
            var       loadEntity = new Entity.ETLLoad
            {
                Table = tableName
            };

            if (!String.IsNullOrEmpty(preCmd))
            {
                stopwatch.Restart();
                await sqlMapper.ExecuteAsync(new RequestContext
                {
                    RealSql = preCmd,
                    Request = queryParams
                });

                stopwatch.Stop();
                loadEntity.PreCommand = new Entity.ETLDbCommand
                {
                    Command   = preCmd,
                    Paramters = queryParams,
                    Taken     = stopwatch.ElapsedMilliseconds
                };
            }

            using (IBatchInsert batchInsert = BatchInsertFactory.Create(sqlMapper, dbProvider))
            {
                InitColumnMapping(batchInsert, context);
                batchInsert.Table = batchTable;
                stopwatch.Restart();
                await batchInsert.InsertAsync();

                stopwatch.Stop();
                loadEntity.Size  = batchTable.Rows.Count;
                loadEntity.Taken = stopwatch.ElapsedMilliseconds;
                _logger.LogWarning($"Build:{context.BuildKey},BatchInsert.Size:{loadEntity.Size},Taken:{loadEntity.Taken}ms!");
            }
            if (context.Build.Paramters.Value(POST_COMMAND, out string postCmd) && !String.IsNullOrEmpty(postCmd))
            {
                stopwatch.Restart();
                await sqlMapper.ExecuteAsync(new RequestContext
                {
                    RealSql = postCmd,
                    Request = queryParams
                });

                stopwatch.Stop();
                loadEntity.PostCommand = new Entity.ETLDbCommand
                {
                    Command   = postCmd,
                    Paramters = queryParams,
                    Taken     = stopwatch.ElapsedMilliseconds
                };
            }
            await etlRepository.Load(_project.GetETKTaskId(), loadEntity);
        }