public ElasticReader(
IConnectionContext context,
Field[] fields,
IElasticLowLevelClient client,
IRowFactory rowFactory,
ReadFrom readFrom
)
{
_context = context;
_fields = fields;
_fieldNames = fields.Select(f => _readFrom == ReadFrom.Input ? f.Name : f.Alias.ToLower()).ToArray();
_client = client;
_rowFactory = rowFactory;
_readFrom = readFrom;
_typeName = readFrom == ReadFrom.Input ? context.Entity.Name : context.Entity.Alias.ToLower();
_context.Entity.ReadSize = _context.Entity.ReadSize == 0 ? DefaultSize : _context.Entity.ReadSize;
if (_context.Entity.ReadSize > ElasticsearchDefaultSizeLimit)
{
_context.Warn("Elasticsearch's default size limit is 10000. {0} may be too high.", _context.Entity.ReadSize);
}
_version = ElasticVersionParser.ParseVersion(_context);
}
public IEnumerable <Field> GetFields(string name)
{
var version = ElasticVersionParser.ParseVersion(_input);
ElasticsearchResponse <DynamicResponse> response;
if (version.Major >= 7)
{
response = _client.IndicesGetMapping <DynamicResponse>(_index, name, qs => qs.AddQueryString("include_type_name", "true"));
}
else
{
response = _client.IndicesGetMapping <DynamicResponse>(_index, name);
}
if (response.Success)
{
var properties = response.Body[_index]["mappings"][name]["properties"] as ElasticsearchDynamicValue;
if (properties != null && properties.HasValue)
{
return(PropertiesToFields(name, properties.Value as IDictionary <string, object>));
}
_input.Error("Could not find properties for index {0} type {1}.", _index, name);
}
else
{
_input.Error(response.ToString());
}
return(Enumerable.Empty <Field>());
}
public IEnumerable <Entity> GetEntities()
{
var version = ElasticVersionParser.ParseVersion(_input);
ElasticsearchResponse <DynamicResponse> response;
if (version.Major >= 7)
{
response = _client.IndicesGetMapping <DynamicResponse>(_index, "_all", qs => qs.AddQueryString("include_type_name", "true"));
}
else
{
response = _client.IndicesGetMapping <DynamicResponse>(_index, "_all");
}
if (response.Success)
{
var mappings = response.Body[_index]["mappings"] as ElasticsearchDynamicValue;
if (mappings != null && mappings.HasValue)
{
var types = mappings.Value as IDictionary <string, object>;
if (types != null)
{
foreach (var pair in types)
{
var e = new Entity {
Name = pair.Key
};
var attributes = pair.Value as IDictionary <string, object>;
if (attributes != null && attributes.ContainsKey("properties"))
{
e.Fields = PropertiesToFields(pair.Key, attributes["properties"] as IDictionary <string, object>).ToList();
}
else
{
_input.Error("Could not find properties for index {0} type {1}.", _input, pair.Key);
}
yield return(e);
}
}
else
{
_input.Error("Could not find types in index {0}.", _index);
}
}
else
{
_input.Error("Could not find mappings for index {0}.", _index);
}
}
else
{
_input.Error(response.ToString());
}
}
public ActionResponse Execute()
{
_context.Warn("Initializing");
var version = ElasticVersionParser.ParseVersion(_context);
var properties = new Dictionary <string, object> {
{ "properties", GetFields() }
};
var typeName = _context.Entity.Alias.ToLower();
var body = new Dictionary <string, object> {
{ typeName, properties }
};
var json = JsonConvert.SerializeObject(body);
ElasticsearchResponse <DynamicResponse> elasticResponse;
if (version.Major >= 7)
{
elasticResponse = _client.IndicesPutMapping <DynamicResponse>(_context.Connection.Index, typeName, json, qs => qs.AddQueryString("include_type_name", "true"));
}
else
{
elasticResponse = _client.IndicesPutMapping <DynamicResponse>(_context.Connection.Index, typeName, json);
}
var response = new ActionResponse(
elasticResponse.HttpStatusCode ?? 500,
elasticResponse.ServerError == null ? string.Empty : elasticResponse.ServerError.Error.Reason ?? string.Empty
)
{
Action = new Configuration.Action()
{
Type = "internal",
ErrorMode = "continue",
Description = $"Initialize {typeName} entity."
}
};
return(response);
}
private Dictionary <string, object> GetFields()
{
var version = ElasticVersionParser.ParseVersion(_context);
var fields = new Dictionary <string, object>();
foreach (var field in _context.OutputFields)
{
var alias = field.Alias.ToLower();
var searchType = _context.Process.SearchTypes.First(st => st.Name == field.SearchType);
var analyzer = searchType.Analyzer;
var type = TranslateType(field.Type);
if (field.Type.Equals("string"))
{
// by default, searchType.Type defers, but on occassion (e.g. geo_point), it takes over
type = searchType.Type == "defer" ? type : searchType.Type;
if (_analyzers.Contains(analyzer))
{
// handle things that are not analyzed
if (analyzer.Equals(string.Empty))
{
if (type.Equals("geo_point"))
{
fields[alias] = new Dictionary <string, object> {
{ "properties", new Dictionary <string, object> {
{ "location", new Dictionary <string, object> {
{ "type", "geo_point" }
} }
} }
};
}
else
{
fields[alias] = new Dictionary <string, object> {
{ "type", version.Major >= 5 ? "keyword" : "string" },
{ "store", searchType.Store }
};
}
}
else
{
if (version.Major >= 5)
{
// version 5+ use keyword and text types instead of string
if (type == "string")
{
if (analyzer == "keyword")
{
fields[alias] = new Dictionary <string, object> {
{ "type", "keyword" }
};
}
else
{
fields[alias] = new Dictionary <string, object> {
{ "type", "text" },
{ "analyzer", analyzer },
{ "store", searchType.Store }
};
}
}
else
{
fields[alias] = new Dictionary <string, object> {
{ "type", type },
{ "analyzer", analyzer }
};
}
}
else // versions prior to 5 uses string types and keyword analyzer
{
fields[alias] = new Dictionary <string, object> {
{ "type", type },
{ "analyzer", analyzer },
{ "store", searchType.Store }
};
}
}
}
else
{
//TODO: MOVE THIS INTO VALIDATION
_context.Warn("Analyzer '{0}' specified in search type '{1}' is not supported. Please use a built-in analyzer for Elasticsearch.", analyzer, field.SearchType);
if (!fields.ContainsKey(alias))
{
fields[alias] = new Dictionary <string, object> {
{ "type", type }
};
}
}
}
else
{
fields[alias] = new Dictionary <string, object> {
{ "type", type }
};
}
}
if (!fields.ContainsKey("tflbatchid"))
{
fields.Add("tflbatchid", new Dictionary <string, object> {
{ "type", "integer" }
});
}
return(fields);
}
