public async Task Build(BuildContext context)
{
ESOptions esOptions = InitOptions(context);
var etlRepository = _pluginManager.Resolve <IETLTaskRepository>(_project.GetETLRepository());
var dataSource = context.GetDataSource <ExtractDataSource>();
var loadEntity = new Entity.ETLLoad
{
Size = 0,
Paramters = new Dictionary <String, object>
{
{ "Task", "LoadToES" },
{ "Index", esOptions.Index },
{ "Type", esOptions.TypeName }
}
};
if (dataSource.TransformData.Rows.Count == 0)
{
await etlRepository.Load(_project.GetETKTaskId(), loadEntity);
return;
}
Stopwatch stopwatch = Stopwatch.StartNew();
#region InitColumnMapping
var colMappings = InitColumnMapping(context);
if (colMappings != null)
{
foreach (var colMapping in colMappings)
{
if (dataSource.TransformData.Columns.ContainsKey(colMapping.Column))
{
dataSource.TransformData.Columns[colMapping.Column].Name = colMapping.Mapping;
}
}
}
#endregion
#region BatchInsert
var esClient = GetElasticClient(esOptions);
var list = new List <Dictionary <String, object> >();
foreach (var row in dataSource.TransformData.Rows)
{
var item = new Dictionary <String, object>();
foreach (var cellKV in row.Cells)
{
var cellVal = cellKV.Value;
item.Add(cellVal.Column.Name, cellVal.Value);
}
list.Add(item);
}
var indexExResp = await esClient.IndexExistsAsync(esOptions.Index);
if (!indexExResp.Exists)
{
var createIndexResp = await esClient.CreateIndexAsync(esOptions.Index);
}
var esSyncResp = await esClient.BulkAsync((bulkRequest) =>
{
var bulkReqDesc = bulkRequest
.Index(esOptions.Index)
.Type(esOptions.TypeName);
if (context.Build.Paramters.Value(ID_MAPPING, out string es_id))
{
return(bulkReqDesc.IndexMany(list, (bulkIdxDesc, item) =>
{
var idVal = item[es_id].ToString();
return bulkIdxDesc.Id(idVal);
}
));
}
return(bulkReqDesc.IndexMany(list));
}
);
if (esSyncResp.Errors || !esSyncResp.IsValid)
{
_logger.LogError($"ES.ERRORS:{esSyncResp.DebugInformation}");
throw new SmartCodeException($"ES.ERRORS:{esSyncResp.DebugInformation}");
}
stopwatch.Stop();
loadEntity.Size = dataSource.TransformData.Rows.Count;
loadEntity.Taken = stopwatch.ElapsedMilliseconds;
#endregion
await etlRepository.Load(_project.GetETKTaskId(), loadEntity);
}
public async Task Build(BuildContext context)
{
context.Build.Paramters.EnsureValue(TABLE_NAME, out string tableName);
context.Build.Paramters.EnsureValue(DB_PROVIDER, out DbProvider dbProvider);
var etlRepository = _pluginManager.Resolve <IETLRepository>(_project.GetETLRepository());
var dataSource = context.GetDataSource <ExtractDataSource>();
if (dataSource.TransformData.Rows.Count == 0)
{
await etlRepository.Load(_project.GetETKTaskId(), new Entity.ETLLoad
{
Size = 0,
Table = tableName
});
return;
}
var batchTable = dataSource.TransformData;
batchTable.Name = tableName;
var sqlMapper = GetSqlMapper(context);
context.Build.Paramters.Value(PRE_COMMAND, out string preCmd);
var lastExtract = _project.GetETLLastExtract();
var queryParams = new Dictionary <string, object>
{
{ "LastMaxId", lastExtract.MaxId },
{ "LastQueryTime", lastExtract.QueryTime },
{ "LastMaxModifyTime", lastExtract.MaxModifyTime },
};
Stopwatch stopwatch = Stopwatch.StartNew();
var loadEntity = new Entity.ETLLoad
{
Table = tableName
};
if (!String.IsNullOrEmpty(preCmd))
{
stopwatch.Restart();
await sqlMapper.ExecuteAsync(new RequestContext
{
RealSql = preCmd,
Request = queryParams
});
stopwatch.Stop();
loadEntity.PreCommand = new Entity.ETLDbCommand
{
Command = preCmd,
Paramters = queryParams,
Taken = stopwatch.ElapsedMilliseconds
};
}
using (IBatchInsert batchInsert = BatchInsertFactory.Create(sqlMapper, dbProvider))
{
InitColumnMapping(batchInsert, context);
batchInsert.Table = batchTable;
stopwatch.Restart();
await batchInsert.InsertAsync();
stopwatch.Stop();
loadEntity.Size = batchTable.Rows.Count;
loadEntity.Taken = stopwatch.ElapsedMilliseconds;
_logger.LogWarning($"Build:{context.BuildKey},BatchInsert.Size:{loadEntity.Size},Taken:{loadEntity.Taken}ms!");
}
if (context.Build.Paramters.Value(POST_COMMAND, out string postCmd) && !String.IsNullOrEmpty(postCmd))
{
stopwatch.Restart();
await sqlMapper.ExecuteAsync(new RequestContext
{
RealSql = postCmd,
Request = queryParams
});
stopwatch.Stop();
loadEntity.PostCommand = new Entity.ETLDbCommand
{
Command = postCmd,
Paramters = queryParams,
Taken = stopwatch.ElapsedMilliseconds
};
}
await etlRepository.Load(_project.GetETKTaskId(), loadEntity);
}
public async Task Build(BuildContext context)
{
ESOptions esOptions = InitOptions(context);
var etlRepository = _pluginManager.Resolve <IETLTaskRepository>(_project.GetETLRepository());
var dataSource = context.GetExtractData <ExtractDictionaryData>();
var loadEntity = new Entity.ETLLoad
{
Size = 0,
Parameters = new Dictionary <String, object>
{
{ "Task", "LoadToES" },
{ "Index", esOptions.Index },
{ "Type", esOptions.TypeName }
}
};
if (dataSource.TransformData.Count == 0)
{
await etlRepository.Load(_project.GetETKTaskId(), loadEntity);
return;
}
Stopwatch stopwatch = Stopwatch.StartNew();
#region InitColumnMapping
var colMapping = new Dictionary <string, string>();
if (context.Build.Parameters.Value(COLUMN_MAPPING, out IEnumerable colMapps))
{
foreach (IDictionary <object, object> colMappingKV in colMapps)
{
colMappingKV.EnsureValue("Column", out string colName);
colMappingKV.EnsureValue("Mapping", out string mapping);
colMapping.Add(colName, mapping);
}
}
dataSource.TransformData = dataSource.TransformData.Select((dic) =>
{
IDictionary <string, object> newItem = new Dictionary <string, object>();
foreach (KeyValuePair <string, object> item in dic)
{
var itemKey = item.Key;
if (colMapps != null)
{
if (colMapping.TryGetValue(itemKey, out string mapping))
{
itemKey = mapping;
}
}
newItem.Add(itemKey, item.Value);
}
return(newItem);
}).ToList();
#endregion
#region BatchInsert
var esClient = GetElasticClient(esOptions);
var indexExResp = await esClient.IndexExistsAsync(esOptions.Index);
if (!indexExResp.Exists)
{
var createIndexResp = await esClient.CreateIndexAsync(esOptions.Index);
}
var esSyncResp = await esClient.BulkAsync((bulkRequest) =>
{
var bulkReqDesc = bulkRequest
.Index(esOptions.Index)
.Type(esOptions.TypeName);
if (context.Build.Parameters.Value(ID_MAPPING, out string es_id))
{
return(bulkReqDesc.IndexMany(dataSource.TransformData, (bulkIdxDesc, item) =>
{
var idVal = item[es_id].ToString();
return bulkIdxDesc.Id(idVal);
}
));
}
return(bulkReqDesc.IndexMany(dataSource.TransformData));
}
);
if (esSyncResp.Errors || !esSyncResp.IsValid)
{
_logger.LogError($"ES.ERRORS:{esSyncResp.DebugInformation}");
throw new SmartCodeException($"ES.ERRORS:{esSyncResp.DebugInformation}");
}
stopwatch.Stop();
loadEntity.Size = dataSource.TransformData.Count;
loadEntity.Taken = stopwatch.ElapsedMilliseconds;
#endregion
await etlRepository.Load(_project.GetETKTaskId(), loadEntity);
}
