public ElasticReader( IConnectionContext context, Field[] fields, IElasticLowLevelClient client, IRowFactory rowFactory, ReadFrom readFrom ) { _context = context; _fields = fields; _fieldNames = fields.Select(f => _readFrom == ReadFrom.Input ? f.Name : f.Alias.ToLower()).ToArray(); _client = client; _rowFactory = rowFactory; _readFrom = readFrom; _typeName = readFrom == ReadFrom.Input ? context.Entity.Name : context.Entity.Alias.ToLower(); _context.Entity.ReadSize = _context.Entity.ReadSize == 0 ? DefaultSize : _context.Entity.ReadSize; if (_context.Entity.ReadSize > ElasticsearchDefaultSizeLimit) { _context.Warn("Elasticsearch's default size limit is 10000. {0} may be too high.", _context.Entity.ReadSize); } _version = ElasticVersionParser.ParseVersion(_context); }
public IEnumerable <Field> GetFields(string name) { var version = ElasticVersionParser.ParseVersion(_input); ElasticsearchResponse <DynamicResponse> response; if (version.Major >= 7) { response = _client.IndicesGetMapping <DynamicResponse>(_index, name, qs => qs.AddQueryString("include_type_name", "true")); } else { response = _client.IndicesGetMapping <DynamicResponse>(_index, name); } if (response.Success) { var properties = response.Body[_index]["mappings"][name]["properties"] as ElasticsearchDynamicValue; if (properties != null && properties.HasValue) { return(PropertiesToFields(name, properties.Value as IDictionary <string, object>)); } _input.Error("Could not find properties for index {0} type {1}.", _index, name); } else { _input.Error(response.ToString()); } return(Enumerable.Empty <Field>()); }
public IEnumerable <Entity> GetEntities() { var version = ElasticVersionParser.ParseVersion(_input); ElasticsearchResponse <DynamicResponse> response; if (version.Major >= 7) { response = _client.IndicesGetMapping <DynamicResponse>(_index, "_all", qs => qs.AddQueryString("include_type_name", "true")); } else { response = _client.IndicesGetMapping <DynamicResponse>(_index, "_all"); } if (response.Success) { var mappings = response.Body[_index]["mappings"] as ElasticsearchDynamicValue; if (mappings != null && mappings.HasValue) { var types = mappings.Value as IDictionary <string, object>; if (types != null) { foreach (var pair in types) { var e = new Entity { Name = pair.Key }; var attributes = pair.Value as IDictionary <string, object>; if (attributes != null && attributes.ContainsKey("properties")) { e.Fields = PropertiesToFields(pair.Key, attributes["properties"] as IDictionary <string, object>).ToList(); } else { _input.Error("Could not find properties for index {0} type {1}.", _input, pair.Key); } yield return(e); } } else { _input.Error("Could not find types in index {0}.", _index); } } else { _input.Error("Could not find mappings for index {0}.", _index); } } else { _input.Error(response.ToString()); } }
public ActionResponse Execute() { _context.Warn("Initializing"); var version = ElasticVersionParser.ParseVersion(_context); var properties = new Dictionary <string, object> { { "properties", GetFields() } }; var typeName = _context.Entity.Alias.ToLower(); var body = new Dictionary <string, object> { { typeName, properties } }; var json = JsonConvert.SerializeObject(body); ElasticsearchResponse <DynamicResponse> elasticResponse; if (version.Major >= 7) { elasticResponse = _client.IndicesPutMapping <DynamicResponse>(_context.Connection.Index, typeName, json, qs => qs.AddQueryString("include_type_name", "true")); } else { elasticResponse = _client.IndicesPutMapping <DynamicResponse>(_context.Connection.Index, typeName, json); } var response = new ActionResponse( elasticResponse.HttpStatusCode ?? 500, elasticResponse.ServerError == null ? string.Empty : elasticResponse.ServerError.Error.Reason ?? string.Empty ) { Action = new Configuration.Action() { Type = "internal", ErrorMode = "continue", Description = $"Initialize {typeName} entity." } }; return(response); }
private Dictionary <string, object> GetFields() { var version = ElasticVersionParser.ParseVersion(_context); var fields = new Dictionary <string, object>(); foreach (var field in _context.OutputFields) { var alias = field.Alias.ToLower(); var searchType = _context.Process.SearchTypes.First(st => st.Name == field.SearchType); var analyzer = searchType.Analyzer; var type = TranslateType(field.Type); if (field.Type.Equals("string")) { // by default, searchType.Type defers, but on occassion (e.g. geo_point), it takes over type = searchType.Type == "defer" ? type : searchType.Type; if (_analyzers.Contains(analyzer)) { // handle things that are not analyzed if (analyzer.Equals(string.Empty)) { if (type.Equals("geo_point")) { fields[alias] = new Dictionary <string, object> { { "properties", new Dictionary <string, object> { { "location", new Dictionary <string, object> { { "type", "geo_point" } } } } } }; } else { fields[alias] = new Dictionary <string, object> { { "type", version.Major >= 5 ? "keyword" : "string" }, { "store", searchType.Store } }; } } else { if (version.Major >= 5) { // version 5+ use keyword and text types instead of string if (type == "string") { if (analyzer == "keyword") { fields[alias] = new Dictionary <string, object> { { "type", "keyword" } }; } else { fields[alias] = new Dictionary <string, object> { { "type", "text" }, { "analyzer", analyzer }, { "store", searchType.Store } }; } } else { fields[alias] = new Dictionary <string, object> { { "type", type }, { "analyzer", analyzer } }; } } else // versions prior to 5 uses string types and keyword analyzer { fields[alias] = new Dictionary <string, object> { { "type", type }, { "analyzer", analyzer }, { "store", searchType.Store } }; } } } else { //TODO: MOVE THIS INTO VALIDATION _context.Warn("Analyzer '{0}' specified in search type '{1}' is not supported. Please use a built-in analyzer for Elasticsearch.", analyzer, field.SearchType); if (!fields.ContainsKey(alias)) { fields[alias] = new Dictionary <string, object> { { "type", type } }; } } } else { fields[alias] = new Dictionary <string, object> { { "type", type } }; } } if (!fields.ContainsKey("tflbatchid")) { fields.Add("tflbatchid", new Dictionary <string, object> { { "type", "integer" } }); } return(fields); }