private IElasticClient GetElasticClient(ESOptions esOptions)
{
Uri node = new Uri(esOptions.Host);
ConnectionSettings settings = new ConnectionSettings(node);
settings.DefaultIndex(esOptions.Index);
settings.DefaultTypeName(esOptions.TypeName);
#region Auth
if (!String.IsNullOrEmpty(esOptions.UserName))
{
settings.BasicAuthentication(esOptions.UserName, esOptions.Password);
}
if (!String.IsNullOrEmpty(esOptions.Cert))
{
settings.ClientCertificate(esOptions.Cert);
}
#endregion
return(new ElasticClient(settings));
}
public async Task Build(BuildContext context)
{
ESOptions esOptions = InitOptions(context);
var etlRepository = _pluginManager.Resolve <IETLTaskRepository>(_project.GetETLRepository());
var dataSource = context.GetDataSource <ExtractDataSource>();
var loadEntity = new Entity.ETLLoad
{
Size = 0,
Paramters = new Dictionary <String, object>
{
{ "Task", "LoadToES" },
{ "Index", esOptions.Index },
{ "Type", esOptions.TypeName }
}
};
if (dataSource.TransformData.Rows.Count == 0)
{
await etlRepository.Load(_project.GetETKTaskId(), loadEntity);
return;
}
Stopwatch stopwatch = Stopwatch.StartNew();
#region InitColumnMapping
var colMappings = InitColumnMapping(context);
if (colMappings != null)
{
foreach (var colMapping in colMappings)
{
if (dataSource.TransformData.Columns.ContainsKey(colMapping.Column))
{
dataSource.TransformData.Columns[colMapping.Column].Name = colMapping.Mapping;
}
}
}
#endregion
#region BatchInsert
var esClient = GetElasticClient(esOptions);
var list = new List <Dictionary <String, object> >();
foreach (var row in dataSource.TransformData.Rows)
{
var item = new Dictionary <String, object>();
foreach (var cellKV in row.Cells)
{
var cellVal = cellKV.Value;
item.Add(cellVal.Column.Name, cellVal.Value);
}
list.Add(item);
}
var indexExResp = await esClient.IndexExistsAsync(esOptions.Index);
if (!indexExResp.Exists)
{
var createIndexResp = await esClient.CreateIndexAsync(esOptions.Index);
}
var esSyncResp = await esClient.BulkAsync((bulkRequest) =>
{
var bulkReqDesc = bulkRequest
.Index(esOptions.Index)
.Type(esOptions.TypeName);
if (context.Build.Paramters.Value(ID_MAPPING, out string es_id))
{
return(bulkReqDesc.IndexMany(list, (bulkIdxDesc, item) =>
{
var idVal = item[es_id].ToString();
return bulkIdxDesc.Id(idVal);
}
));
}
return(bulkReqDesc.IndexMany(list));
}
);
if (esSyncResp.Errors || !esSyncResp.IsValid)
{
_logger.LogError($"ES.ERRORS:{esSyncResp.DebugInformation}");
throw new SmartCodeException($"ES.ERRORS:{esSyncResp.DebugInformation}");
}
stopwatch.Stop();
loadEntity.Size = dataSource.TransformData.Rows.Count;
loadEntity.Taken = stopwatch.ElapsedMilliseconds;
#endregion
await etlRepository.Load(_project.GetETKTaskId(), loadEntity);
}
public async Task Build(BuildContext context)
{
ESOptions esOptions = InitOptions(context);
var etlRepository = _pluginManager.Resolve <IETLTaskRepository>(_project.GetETLRepository());
var dataSource = context.GetExtractData <ExtractDictionaryData>();
var loadEntity = new Entity.ETLLoad
{
Size = 0,
Parameters = new Dictionary <String, object>
{
{ "Task", "LoadToES" },
{ "Index", esOptions.Index },
{ "Type", esOptions.TypeName }
}
};
if (dataSource.TransformData.Count == 0)
{
await etlRepository.Load(_project.GetETKTaskId(), loadEntity);
return;
}
Stopwatch stopwatch = Stopwatch.StartNew();
#region InitColumnMapping
var colMapping = new Dictionary <string, string>();
if (context.Build.Parameters.Value(COLUMN_MAPPING, out IEnumerable colMapps))
{
foreach (IDictionary <object, object> colMappingKV in colMapps)
{
colMappingKV.EnsureValue("Column", out string colName);
colMappingKV.EnsureValue("Mapping", out string mapping);
colMapping.Add(colName, mapping);
}
}
dataSource.TransformData = dataSource.TransformData.Select((dic) =>
{
IDictionary <string, object> newItem = new Dictionary <string, object>();
foreach (KeyValuePair <string, object> item in dic)
{
var itemKey = item.Key;
if (colMapps != null)
{
if (colMapping.TryGetValue(itemKey, out string mapping))
{
itemKey = mapping;
}
}
newItem.Add(itemKey, item.Value);
}
return(newItem);
}).ToList();
#endregion
#region BatchInsert
var esClient = GetElasticClient(esOptions);
var indexExResp = await esClient.IndexExistsAsync(esOptions.Index);
if (!indexExResp.Exists)
{
var createIndexResp = await esClient.CreateIndexAsync(esOptions.Index);
}
var esSyncResp = await esClient.BulkAsync((bulkRequest) =>
{
var bulkReqDesc = bulkRequest
.Index(esOptions.Index)
.Type(esOptions.TypeName);
if (context.Build.Parameters.Value(ID_MAPPING, out string es_id))
{
return(bulkReqDesc.IndexMany(dataSource.TransformData, (bulkIdxDesc, item) =>
{
var idVal = item[es_id].ToString();
return bulkIdxDesc.Id(idVal);
}
));
}
return(bulkReqDesc.IndexMany(dataSource.TransformData));
}
);
if (esSyncResp.Errors || !esSyncResp.IsValid)
{
_logger.LogError($"ES.ERRORS:{esSyncResp.DebugInformation}");
throw new SmartCodeException($"ES.ERRORS:{esSyncResp.DebugInformation}");
}
stopwatch.Stop();
loadEntity.Size = dataSource.TransformData.Count;
loadEntity.Taken = stopwatch.ElapsedMilliseconds;
#endregion
await etlRepository.Load(_project.GetETKTaskId(), loadEntity);
}
