public RawResults <TRoot> GetRawResults <TRoot>(RequestBase query, string language, string indexName = null)
{
if (indexName == null)
{
indexName = _elasticSearchSettings.GetDefaultIndexName(language);
}
Logger.Information($"Index:\n{indexName}\n");
Logger.Information($"Query:\n{query?.ToString(Formatting.Indented)}\n");
var uri = GetSearchEndpoint(indexName);
JsonReader response = GetResponse(query, uri, out string rawJsonResult);
var serializer = new JsonSerializer
{
ReferenceLoopHandling = ReferenceLoopHandling.Ignore,
NullValueHandling = NullValueHandling.Ignore
};
TRoot rawResults = response == null
? default
: serializer.Deserialize <TRoot>(response);
return(new RawResults <TRoot>
{
RootObject = rawResults,
RawJson = rawJsonResult
});
}
/// <summary>
/// Gets all property mappings for the configured index and the supplied type
/// </summary>
internal static async Task <IndexMapping> GetIndexMappingAsync(Type type, string language, string index)
{
if (String.IsNullOrEmpty(index))
{
index = ElasticSearchSettings.GetDefaultIndexName(language);
}
index = index.ToLower();
string typeName = type.GetTypeName();
string mappingUri = GetMappingUri(index, typeName);
IndexMapping mappings;
Logger.Debug($"GetIndexMapping for: {typeName}. Uri: {mappingUri}");
try
{
string mappingJson = await HttpClientHelper.GetStringAsync(new Uri(mappingUri));
mappings = BuildIndexMapping(mappingJson, index, typeName);
}
catch (Exception ex)
{
Logger.Debug("Failed to get existing mapping from uri '" + mappingUri + "'", ex);
mappings = new IndexMapping();
}
if (mappings.Properties == null)
{
mappings.Properties = new Dictionary <string, IndexMappingProperty>();
}
return(mappings);
}
/// <summary>
/// Gets all property mappings for the configured index and the supplied type
/// </summary>
internal IndexMapping GetIndexMapping(Type type, string language, string index)
{
if (String.IsNullOrEmpty(index))
{
index = _settings.GetDefaultIndexName(language);
}
string typeName = type.GetTypeName();
string mappingUri = GetMappingUri(index, typeName);
IndexMapping mappings;
_logger.Debug($"GetIndexMapping for: {typeName}. Uri: {mappingUri}");
try
{
mappings = BuildIndexMapping(_httpClientHelper.GetString(new Uri(mappingUri)), index, typeName);
}
catch (Exception ex)
{
_logger.Debug("Failed to get existing mapping from uri '" + mappingUri + "'", ex);
mappings = new IndexMapping();
}
if (mappings.Properties == null)
{
mappings.Properties = new Dictionary <string, IndexMappingProperty>();
}
return(mappings);
}
public void Delete(string id, string language, Type type, string indexName = null)
{
if (indexName == null)
{
indexName = _settings.GetDefaultIndexName(language);
}
var uri = $"{_settings.Host}/{indexName}/{type.GetTypeName()}/{id}";
var exists = _httpClientHelper.Head(new Uri(uri)) == HttpStatusCode.OK;
if (exists)
{
_httpClientHelper.Delete(new Uri(uri));
Refresh(language);
}
}
private string GetIndexName(string language)
{
if (!String.IsNullOrEmpty(CustomIndexName))
{
return(_settings.GetCustomIndexName(CustomIndexName, language));
}
return(_settings.GetDefaultIndexName(language));
}
public BulkBatchResult BulkUpdate(IEnumerable <IContent> contents, Action <string> logger, string indexName = null)
{
List <IContent> contentList = contents.ToList();
logger = logger ?? delegate { };
var before = contentList.Count;
logger("Filtering away content of excluded types and content with property HideFromSearch enabled...");
contentList.RemoveAll(ShouldHideFromSearch);
contentList.RemoveAll(IsExludedType);
logger($"Filtered away content of excluded types and content with property HideFromSearch enabled... {before - contentList.Count} of {before} items removed. Next will IContent be converted to indexable items and added to indexed. Depending on number of IContent items this is a time-consuming task.");
var operations =
contentList.Select(
content =>
{
var language = GetLanguage(content);
var index = String.IsNullOrWhiteSpace(indexName)
? _elasticSearchSettings.GetDefaultIndexName(language)
: _elasticSearchSettings.GetCustomIndexName(indexName, language);
return(new BulkOperation(
content.AsIndexItem(),
Operation.Index,
GetLanguage(content),
typeof(IndexItem),
content.ContentLink.ToReferenceWithoutVersion().ToString(),
index));
}
)
.Where(b => b.Data != null)
.ToList();
logger($"Initializing bulk operation... Bulk indexing {operations.Count} items");
return(_coreIndexer.Bulk(operations, logger));
}
private void UpdateMappings(IList <LanguageBranch> languages, string indexName = null)
{
// Update mappings
foreach (var language in languages.Select(l => l.LanguageID))
{
try
{
if (String.IsNullOrWhiteSpace(indexName))
{
indexName = _settings.GetDefaultIndexName(language);
}
else
{
indexName = _settings.GetCustomIndexName(indexName, language);
}
_logger.Debug("Index: " + indexName);
var indexing = new Indexing(_settings);
if (!indexing.IndexExists(indexName))
{
throw new Exception("Index does not exist");
}
OnStatusChanged("Updating mapping for index " + indexName);
_coreIndexer.UpdateMapping(typeof(IndexItem), typeof(IndexItem), indexName);
ContentExtensions.CreateAnalyzedMappingsIfNeeded(typeof(IndexItem), language, indexName);
ContentExtensions.CreateDidYouMeanMappingsIfNeeded(typeof(IndexItem), language, indexName);
}
catch (Exception ex)
{
_logger.Warning("Uh oh...", ex);
}
}
}
private string GetIndexName(string languageId, string selectedIndex = null)
{
return(selectedIndex != null
? _elasticSearchSettings.GetCustomIndexName(selectedIndex, languageId)
: _elasticSearchSettings.GetDefaultIndexName(languageId));
}
private void SetupBoosting(QuerySetup setup, QueryRequest request)
{
if (!setup.UseBoosting)
{
return;
}
List <Boost> boosting = GetBoosting(setup.Type, setup.BoostFields);
if (boosting.Count > 0)
{
var searchText = request.Query.SearchText.Replace("*", String.Empty);
if (!TextUtil.IsNumeric(searchText))
{
boosting.RemoveAll(b => b.FieldName.Equals(DefaultFields.Id));
}
request.Query.Bool.Should.AddRange(
boosting.Select(b =>
new MatchWithBoost(b.FieldName,
searchText, b.Weight, setup.Operator)));
}
// Boosting by type
if (setup.BoostTypes.Count > 0)
{
request.Query.Bool.Should.AddRange(
setup.BoostTypes.Select(b =>
new MatchWithBoost(DefaultFields.Types, b.Key.GetTypeName(), b.Value, setup.Operator)));
// Direct match in Type gives higher score than match in Types, hence the +1
request.Query.Bool.Should.AddRange(
setup.BoostTypes.Select(b =>
new MatchWithBoost(DefaultFields.Type, b.Key.GetTypeName(), b.Value + 1, setup.Operator)));
}
if (setup.BoostAncestors.Count > 0)
{
request.Query.Bool.Should.AddRange(
setup.BoostAncestors.Select(b =>
new MatchWithBoost(DefaultFields.Path, b.Key.ToString(), b.Value, setup.Operator)));
request.Query.Bool.Should.AddRange(
setup.BoostAncestors.Select(b =>
new MatchWithBoost(DefaultFields.Id, b.Key.ToString(), b.Value, setup.Operator)));
}
// Best Bets
if (setup.UseBestBets && !String.IsNullOrWhiteSpace(request.Query.SearchText))
{
IEnumerable <string> terms = request.Query.SearchText
.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)
.Select(t => t.Trim().Trim('*'));
var key = setup.IndexName ?? _settings.GetDefaultIndexName(Language.GetLanguageCode(setup.Language));
if (!Conventions.Indexing.BestBets.TryGetValue(key, out var bestBetsForLanguage))
{
return;
}
IEnumerable <BestBet> bestBets = bestBetsForLanguage
.Where(b => b.Terms.Any(t => terms.Contains(t)));
request.Query.Bool.Should.AddRange(
bestBets.Select(_ =>
new MatchWithBoost(
DefaultFields.BestBets, request.Query.SearchText.Trim('*'), BestBetMultiplier, setup.Operator)));
}
}
public void SetSynonyms(string languageId, string analyzer, List <Synonym> synonymsToAdd, string index)
{
if (String.IsNullOrWhiteSpace(index))
{
index = _settings.GetDefaultIndexName(languageId);
}
var indexing = new Indexing(_serverInfoService, _settings, _httpClientHelper);
indexing.Close(index);
try
{
string[] synonymPairs = synonymsToAdd
.Select(s => String.Concat(s.From, s.MultiWord ? "=>" : ",", s.To))
.ToArray();
if (synonymPairs.Length == 0)
{
synonymPairs = new[] { "example_from,example_to" };
}
_logger.Information(
$"Adding {synonymsToAdd.Count} synonyms for language:{languageId} and analyzer:{analyzer}");
if (_logger.IsDebugEnabled())
{
synonymPairs.ToList().ForEach(pair => _logger.Debug(pair));
}
dynamic body = new
{
settings = new
{
analysis = new
{
filter = new
{
ANALYZERTOKEN_synonym_filter = new
{
type = "synonym", synonyms = synonymPairs
}
}
}
}
};
SaveBackup(languageId, index, synonymPairs);
string json = Serialization.Serialize(body);
json = json.Replace("ANALYZERTOKEN", analyzer);
if (_logger.IsDebugEnabled())
{
_logger.Debug("SYNONYM JSON PAYLOAD:\n" + json);
}
var data = Encoding.UTF8.GetBytes(json);
var uri = indexing.GetUri(index, "_settings");
_httpClientHelper.Put(uri, data);
}
catch (Exception ex)
{
_logger.Error($"Failure adding {synonymsToAdd.Count} synonyms for language:{languageId} and analyzer:{analyzer}", ex);
}
finally
{
indexing.Open(index);
}
}
/// <summary>
/// Creates a bulk-operation to be used in <see cref="CoreIndexer.Bulk(Epinova.ElasticSearch.Core.Models.Bulk.BulkOperation[])"/>.
/// Uses configured index if <paramref name="index"/> is empty.
/// </summary>
internal BulkOperation(object data, Operation operation, string language = null, Type dataType = null, string id = null, string index = null)
{
if (String.IsNullOrWhiteSpace(language) && String.IsNullOrWhiteSpace(index))
{
throw new InvalidOperationException("Either 'language' or 'index' must be specified.");
}
dataType = dataType ?? data.GetType();
id = GetId(id, dataType, data);
// If we have no Types, this is a custom object and we must extract the properties from the data-object.
// Standard IndexItems will already have needed data created by AsIndexItem
if (dataType.GetProperty(DefaultFields.Types) == null)
{
dynamic indexItem = new ExpandoObject();
var dictionary = (IDictionary <string, object>)indexItem;
foreach (var property in data.GetType().GetProperties())
{
try
{
var value = property.GetValue(data);
if (value == null)
{
continue;
}
if (property.PropertyType == typeof(bool) || property.PropertyType == typeof(bool?))
{
value = (bool)value;
}
else if (property.PropertyType.IsEnum)
{
value = (int)value;
}
else if (value is DateTime)
{
// Don't ToString or anything funky here
}
else if (value is decimal dec)
{
value = dec.ToString(DotSeparatorFormat);
}
else if (value is double dbl)
{
value = dbl.ToString(DotSeparatorFormat);
}
else if (value is float flt)
{
value = flt.ToString(DotSeparatorFormat);
}
else if (ArrayHelper.IsArrayCandidate(property))
{
value = ArrayHelper.ToArray(value);
}
else if (Utilities.Mapping.IsNumericType(property.PropertyType))
{
value = value.ToString().Trim('\"');
}
else if (property.PropertyType.IsValueType || property.PropertyType.IsPrimitive)
{
value = value.ToString().Trim('\"');
}
else
{
value = value.ToString().Trim('\"');
}
dictionary[property.Name] = value;
}
catch (Exception ex)
{
Logger.Error("Failed to apply object property: " + property.Name, ex);
}
}
dictionary[DefaultFields.Types] = dataType.GetInheritancHierarchyArray();
if (dataType.GetProperty(DefaultFields.Type) == null)
{
dictionary.Add(DefaultFields.Type, dataType.GetTypeName());
}
data = indexItem;
}
if (String.IsNullOrWhiteSpace(index))
{
index = ElasticSearchSettings.GetDefaultIndexName(language);
}
MetaData = new BulkMetadata
{
Operation = operation,
DataType = dataType,
Type = dataType.GetTypeName(),
Id = id,
IndexCandidate = index.ToLower()
};
Data = data;
}
private string GetIndexName(string language) => !String.IsNullOrEmpty(CustomIndexName) ? _settings.GetCustomIndexName(CustomIndexName, language) : _settings.GetDefaultIndexName(language);
