public RawResults <TRoot> GetRawResults <TRoot>(RequestBase query, string language, string indexName = null) { if (indexName == null) { indexName = _elasticSearchSettings.GetDefaultIndexName(language); } Logger.Information($"Index:\n{indexName}\n"); Logger.Information($"Query:\n{query?.ToString(Formatting.Indented)}\n"); var uri = GetSearchEndpoint(indexName); JsonReader response = GetResponse(query, uri, out string rawJsonResult); var serializer = new JsonSerializer { ReferenceLoopHandling = ReferenceLoopHandling.Ignore, NullValueHandling = NullValueHandling.Ignore }; TRoot rawResults = response == null ? default : serializer.Deserialize <TRoot>(response); return(new RawResults <TRoot> { RootObject = rawResults, RawJson = rawJsonResult }); }
/// <summary> /// Gets all property mappings for the configured index and the supplied type /// </summary> internal static async Task <IndexMapping> GetIndexMappingAsync(Type type, string language, string index) { if (String.IsNullOrEmpty(index)) { index = ElasticSearchSettings.GetDefaultIndexName(language); } index = index.ToLower(); string typeName = type.GetTypeName(); string mappingUri = GetMappingUri(index, typeName); IndexMapping mappings; Logger.Debug($"GetIndexMapping for: {typeName}. Uri: {mappingUri}"); try { string mappingJson = await HttpClientHelper.GetStringAsync(new Uri(mappingUri)); mappings = BuildIndexMapping(mappingJson, index, typeName); } catch (Exception ex) { Logger.Debug("Failed to get existing mapping from uri '" + mappingUri + "'", ex); mappings = new IndexMapping(); } if (mappings.Properties == null) { mappings.Properties = new Dictionary <string, IndexMappingProperty>(); } return(mappings); }
/// <summary> /// Gets all property mappings for the configured index and the supplied type /// </summary> internal IndexMapping GetIndexMapping(Type type, string language, string index) { if (String.IsNullOrEmpty(index)) { index = _settings.GetDefaultIndexName(language); } string typeName = type.GetTypeName(); string mappingUri = GetMappingUri(index, typeName); IndexMapping mappings; _logger.Debug($"GetIndexMapping for: {typeName}. Uri: {mappingUri}"); try { mappings = BuildIndexMapping(_httpClientHelper.GetString(new Uri(mappingUri)), index, typeName); } catch (Exception ex) { _logger.Debug("Failed to get existing mapping from uri '" + mappingUri + "'", ex); mappings = new IndexMapping(); } if (mappings.Properties == null) { mappings.Properties = new Dictionary <string, IndexMappingProperty>(); } return(mappings); }
public void Delete(string id, string language, Type type, string indexName = null) { if (indexName == null) { indexName = _settings.GetDefaultIndexName(language); } var uri = $"{_settings.Host}/{indexName}/{type.GetTypeName()}/{id}"; var exists = _httpClientHelper.Head(new Uri(uri)) == HttpStatusCode.OK; if (exists) { _httpClientHelper.Delete(new Uri(uri)); Refresh(language); } }
private string GetIndexName(string language) { if (!String.IsNullOrEmpty(CustomIndexName)) { return(_settings.GetCustomIndexName(CustomIndexName, language)); } return(_settings.GetDefaultIndexName(language)); }
public BulkBatchResult BulkUpdate(IEnumerable <IContent> contents, Action <string> logger, string indexName = null) { List <IContent> contentList = contents.ToList(); logger = logger ?? delegate { }; var before = contentList.Count; logger("Filtering away content of excluded types and content with property HideFromSearch enabled..."); contentList.RemoveAll(ShouldHideFromSearch); contentList.RemoveAll(IsExludedType); logger($"Filtered away content of excluded types and content with property HideFromSearch enabled... {before - contentList.Count} of {before} items removed. Next will IContent be converted to indexable items and added to indexed. Depending on number of IContent items this is a time-consuming task."); var operations = contentList.Select( content => { var language = GetLanguage(content); var index = String.IsNullOrWhiteSpace(indexName) ? _elasticSearchSettings.GetDefaultIndexName(language) : _elasticSearchSettings.GetCustomIndexName(indexName, language); return(new BulkOperation( content.AsIndexItem(), Operation.Index, GetLanguage(content), typeof(IndexItem), content.ContentLink.ToReferenceWithoutVersion().ToString(), index)); } ) .Where(b => b.Data != null) .ToList(); logger($"Initializing bulk operation... Bulk indexing {operations.Count} items"); return(_coreIndexer.Bulk(operations, logger)); }
private void UpdateMappings(IList <LanguageBranch> languages, string indexName = null) { // Update mappings foreach (var language in languages.Select(l => l.LanguageID)) { try { if (String.IsNullOrWhiteSpace(indexName)) { indexName = _settings.GetDefaultIndexName(language); } else { indexName = _settings.GetCustomIndexName(indexName, language); } _logger.Debug("Index: " + indexName); var indexing = new Indexing(_settings); if (!indexing.IndexExists(indexName)) { throw new Exception("Index does not exist"); } OnStatusChanged("Updating mapping for index " + indexName); _coreIndexer.UpdateMapping(typeof(IndexItem), typeof(IndexItem), indexName); ContentExtensions.CreateAnalyzedMappingsIfNeeded(typeof(IndexItem), language, indexName); ContentExtensions.CreateDidYouMeanMappingsIfNeeded(typeof(IndexItem), language, indexName); } catch (Exception ex) { _logger.Warning("Uh oh...", ex); } } }
private string GetIndexName(string languageId, string selectedIndex = null) { return(selectedIndex != null ? _elasticSearchSettings.GetCustomIndexName(selectedIndex, languageId) : _elasticSearchSettings.GetDefaultIndexName(languageId)); }
private void SetupBoosting(QuerySetup setup, QueryRequest request) { if (!setup.UseBoosting) { return; } List <Boost> boosting = GetBoosting(setup.Type, setup.BoostFields); if (boosting.Count > 0) { var searchText = request.Query.SearchText.Replace("*", String.Empty); if (!TextUtil.IsNumeric(searchText)) { boosting.RemoveAll(b => b.FieldName.Equals(DefaultFields.Id)); } request.Query.Bool.Should.AddRange( boosting.Select(b => new MatchWithBoost(b.FieldName, searchText, b.Weight, setup.Operator))); } // Boosting by type if (setup.BoostTypes.Count > 0) { request.Query.Bool.Should.AddRange( setup.BoostTypes.Select(b => new MatchWithBoost(DefaultFields.Types, b.Key.GetTypeName(), b.Value, setup.Operator))); // Direct match in Type gives higher score than match in Types, hence the +1 request.Query.Bool.Should.AddRange( setup.BoostTypes.Select(b => new MatchWithBoost(DefaultFields.Type, b.Key.GetTypeName(), b.Value + 1, setup.Operator))); } if (setup.BoostAncestors.Count > 0) { request.Query.Bool.Should.AddRange( setup.BoostAncestors.Select(b => new MatchWithBoost(DefaultFields.Path, b.Key.ToString(), b.Value, setup.Operator))); request.Query.Bool.Should.AddRange( setup.BoostAncestors.Select(b => new MatchWithBoost(DefaultFields.Id, b.Key.ToString(), b.Value, setup.Operator))); } // Best Bets if (setup.UseBestBets && !String.IsNullOrWhiteSpace(request.Query.SearchText)) { IEnumerable <string> terms = request.Query.SearchText .Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries) .Select(t => t.Trim().Trim('*')); var key = setup.IndexName ?? _settings.GetDefaultIndexName(Language.GetLanguageCode(setup.Language)); if (!Conventions.Indexing.BestBets.TryGetValue(key, out var bestBetsForLanguage)) { return; } IEnumerable <BestBet> bestBets = bestBetsForLanguage .Where(b => b.Terms.Any(t => terms.Contains(t))); request.Query.Bool.Should.AddRange( bestBets.Select(_ => new MatchWithBoost( DefaultFields.BestBets, request.Query.SearchText.Trim('*'), BestBetMultiplier, setup.Operator))); } }
public void SetSynonyms(string languageId, string analyzer, List <Synonym> synonymsToAdd, string index) { if (String.IsNullOrWhiteSpace(index)) { index = _settings.GetDefaultIndexName(languageId); } var indexing = new Indexing(_serverInfoService, _settings, _httpClientHelper); indexing.Close(index); try { string[] synonymPairs = synonymsToAdd .Select(s => String.Concat(s.From, s.MultiWord ? "=>" : ",", s.To)) .ToArray(); if (synonymPairs.Length == 0) { synonymPairs = new[] { "example_from,example_to" }; } _logger.Information( $"Adding {synonymsToAdd.Count} synonyms for language:{languageId} and analyzer:{analyzer}"); if (_logger.IsDebugEnabled()) { synonymPairs.ToList().ForEach(pair => _logger.Debug(pair)); } dynamic body = new { settings = new { analysis = new { filter = new { ANALYZERTOKEN_synonym_filter = new { type = "synonym", synonyms = synonymPairs } } } } }; SaveBackup(languageId, index, synonymPairs); string json = Serialization.Serialize(body); json = json.Replace("ANALYZERTOKEN", analyzer); if (_logger.IsDebugEnabled()) { _logger.Debug("SYNONYM JSON PAYLOAD:\n" + json); } var data = Encoding.UTF8.GetBytes(json); var uri = indexing.GetUri(index, "_settings"); _httpClientHelper.Put(uri, data); } catch (Exception ex) { _logger.Error($"Failure adding {synonymsToAdd.Count} synonyms for language:{languageId} and analyzer:{analyzer}", ex); } finally { indexing.Open(index); } }
/// <summary> /// Creates a bulk-operation to be used in <see cref="CoreIndexer.Bulk(Epinova.ElasticSearch.Core.Models.Bulk.BulkOperation[])"/>. /// Uses configured index if <paramref name="index"/> is empty. /// </summary> internal BulkOperation(object data, Operation operation, string language = null, Type dataType = null, string id = null, string index = null) { if (String.IsNullOrWhiteSpace(language) && String.IsNullOrWhiteSpace(index)) { throw new InvalidOperationException("Either 'language' or 'index' must be specified."); } dataType = dataType ?? data.GetType(); id = GetId(id, dataType, data); // If we have no Types, this is a custom object and we must extract the properties from the data-object. // Standard IndexItems will already have needed data created by AsIndexItem if (dataType.GetProperty(DefaultFields.Types) == null) { dynamic indexItem = new ExpandoObject(); var dictionary = (IDictionary <string, object>)indexItem; foreach (var property in data.GetType().GetProperties()) { try { var value = property.GetValue(data); if (value == null) { continue; } if (property.PropertyType == typeof(bool) || property.PropertyType == typeof(bool?)) { value = (bool)value; } else if (property.PropertyType.IsEnum) { value = (int)value; } else if (value is DateTime) { // Don't ToString or anything funky here } else if (value is decimal dec) { value = dec.ToString(DotSeparatorFormat); } else if (value is double dbl) { value = dbl.ToString(DotSeparatorFormat); } else if (value is float flt) { value = flt.ToString(DotSeparatorFormat); } else if (ArrayHelper.IsArrayCandidate(property)) { value = ArrayHelper.ToArray(value); } else if (Utilities.Mapping.IsNumericType(property.PropertyType)) { value = value.ToString().Trim('\"'); } else if (property.PropertyType.IsValueType || property.PropertyType.IsPrimitive) { value = value.ToString().Trim('\"'); } else { value = value.ToString().Trim('\"'); } dictionary[property.Name] = value; } catch (Exception ex) { Logger.Error("Failed to apply object property: " + property.Name, ex); } } dictionary[DefaultFields.Types] = dataType.GetInheritancHierarchyArray(); if (dataType.GetProperty(DefaultFields.Type) == null) { dictionary.Add(DefaultFields.Type, dataType.GetTypeName()); } data = indexItem; } if (String.IsNullOrWhiteSpace(index)) { index = ElasticSearchSettings.GetDefaultIndexName(language); } MetaData = new BulkMetadata { Operation = operation, DataType = dataType, Type = dataType.GetTypeName(), Id = id, IndexCandidate = index.ToLower() }; Data = data; }
private string GetIndexName(string language) => !String.IsNullOrEmpty(CustomIndexName) ? _settings.GetCustomIndexName(CustomIndexName, language) : _settings.GetDefaultIndexName(language);