/// <summary>
/// Get synonyms
/// </summary>
/// <param name="client"></param>
/// <returns></returns>
protected virtual string[] GetSynonyms(IVulcanClient client)
{
var resolved = new List <string>();
var synonyms = client.GetSynonyms();
if (synonyms != null)
{
foreach (var synonym in synonyms)
{
if (synonym.Value.Value) // bidirectional
{
resolved.Add(synonym.Key + "," + string.Join(",", synonym.Value.Key));
}
else
{
resolved.Add(synonym.Key + " => " + string.Join(",", synonym.Value.Key));
}
}
}
if (resolved.Count == 0)
{
resolved.Add("thisisadummyterm,makesurethesynonymsenabled");
}
return(resolved.ToArray());
}
private void Context_InitComplete(object sender, EventArgs e)
{
IVulcanHandler handler = ServiceLocator.Current.GetInstance <IVulcanHandler>();
// Clear static list so property mapping can be re-created.
handler.DeletedIndices += ((IEnumerable <string> deletedIndices) =>
{
VulcanAttachmentPropertyMapper.AddedMappings.Clear();
});
if (_AttachmentSettings.Service.EnableAttachmentPlugins)
{
//todo: future 5.x support uses https://www.elastic.co/guide/en/elasticsearch/plugins/5.2/ingest-attachment.html
// not which 2.x uses https://www.elastic.co/guide/en/elasticsearch/plugins/5.2/mapper-attachments.html
IVulcanClient client = handler.GetClient(CultureInfo.InvariantCulture);
var info = client.NodesInfo();
if (info?.Nodes?.Any(x => x.Value?.Plugins?.Any(y => string.Compare(y.Name, "mapper-attachments", true) == 0) == true) != true)
{
if (CurrentHostType != HostType.WebApplication ||
(CurrentHostType == HostType.WebApplication && HttpContext.Current?.IsDebuggingEnabled == true))
{
// Only throw exception if not a web application or is a web application with debug turned on
throw new Exception("No attachment plugin found, be sure to install the 'mapper-attachments' plugin on your Elastic Search Server!");
}
}
}
}
/// <summary>
/// Provides quick search, filtered by current user
/// </summary>
/// <param name="client"></param>
/// <param name="searchText">Full text query against analyzed fields and uploaded assets if attachments are indexed.</param>
/// <param name="page"></param>
/// <param name="pageSize"></param>
/// <param name="searchRoots"></param>
/// <param name="includeTypes"></param>
/// <param name="excludeTypes"></param>
/// <param name="buildSearchHit">Can be used to customize how VulcanSearchHit is populated. Default is IVulcanClientExtensions.DefaultBuildSearchHit</param>
/// <returns></returns>
public static VulcanSearchHitList GetSearchHits(this IVulcanClient client,
string searchText,
int page,
int pageSize,
IEnumerable <ContentReference> searchRoots = null,
IEnumerable <Type> includeTypes = null,
IEnumerable <Type> excludeTypes = null,
Func <IHit <IContent>, IContentLoader, VulcanSearchHit> buildSearchHit = null
)
{
QueryContainer searchTextQuery = new QueryContainerDescriptor <IContent>();
// only add query string if query has value
if (!string.IsNullOrWhiteSpace(searchText))
{
searchTextQuery = new QueryContainerDescriptor <IContent>().SimpleQueryString(sqs => sqs
.Fields(f => f
.AllAnalyzed()
.Field($"{MediaContents}.content")
.Field($"{MediaContents}.content_type"))
.Query(searchText)
);
}
searchTextQuery = searchTextQuery.FilterForPublished <IContent>();
return(GetSearchHits(client, searchTextQuery, page, pageSize, searchRoots, includeTypes, excludeTypes, buildSearchHit));
}
/// <summary>
/// Allows for customizations on analyzers and mappings.
/// </summary>
/// <param name="client"></param>
/// <param name="logger"></param>
public static void RunCustomizers(this IVulcanClient client, ILogger logger)
{
// run index updaters first, incase they are creating analyzers the mapping need
foreach (var customizer in Customizers)
{
try
{
var updateResponse = customizer?.CustomIndexUpdater?.Invoke(client);
if (updateResponse?.IsValid == false)
{
logger.Error("Could not update index " + client.IndexName + ": " + updateResponse.DebugInformation);
}
}
catch (NotImplementedException) { }
}
// then run the mappings
foreach (var customizer in Customizers)
{
try
{
var mappingResponse = customizer?.CustomMapper?.Invoke(client);
if (mappingResponse?.IsValid == false)
{
logger.Error("Could not add mapping for index " + client.IndexName + ": " + mappingResponse.DebugInformation);
}
}
catch (NotImplementedException) { }
}
}
/// <summary>
/// Add mapping by string
/// </summary>
/// <param name="typeName"></param>
public static void AddMapping(string typeName)
{
if (AddedMappings?.Contains(typeName) == true)
{
return;
}
try
{
IVulcanClient client = ServiceLocator.Current.GetInstance <IVulcanHandler>().GetClient(CultureInfo.InvariantCulture);
var response = client.Map <object>(m => m.
Index("_all").
Type(typeName).
Properties(props => props.
Attachment(s => s.Name(MediaContents)
.FileField(ff => ff.Name("content").Store().TermVector(Nest.TermVectorOption.WithPositionsOffsets))
))
);
if (!response.IsValid)
{
throw new Exception(response.DebugInformation);
}
AddedMappings.Add(typeName);
}
catch (Exception ex)
{
_Logger.Error("Failed to map attachment field for type: " + typeName, ex);
}
}
/// <summary>
/// Adds full name as search type, and ensures invariant culture for POCO searching.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="client"></param>
/// <param name="searchDescriptor"></param>
/// <returns></returns>
public static ISearchResponse <T> PocoSearch <T>(this IVulcanClient client, Func <SearchDescriptor <T>, SearchDescriptor <T> > searchDescriptor = null) where T : class
{
var tempClient = client.Language == CultureInfo.InvariantCulture ? client : VulcanHandler.Service.GetClient(CultureInfo.InvariantCulture);
SearchDescriptor <T> resolvedDescriptor = searchDescriptor?.Invoke(new SearchDescriptor <T>()) ?? new SearchDescriptor <T>();
resolvedDescriptor = resolvedDescriptor.Type(typeof(T).FullName);
return(tempClient.Search <T>(resolvedDescriptor));
}
/// <summary>
/// Adds full name as search type, and ensures invariant culture for POCO searching.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="client"></param>
/// <param name="searchDescriptor"></param>
/// <param name="alias"></param>
/// <returns></returns>
public static ISearchResponse <T> PocoSearch <T>(this IVulcanClient client, Func <SearchDescriptor <T>, SearchDescriptor <T> > searchDescriptor = null, string alias = null) where T : class
{
VulcanHelper.GuardForNullAlias(ref alias);
var tempClient = client.Language.Equals(CultureInfo.InvariantCulture) ? client : VulcanHandler.Service.GetClient(CultureInfo.InvariantCulture, alias);
var resolvedDescriptor = searchDescriptor?.Invoke(new SearchDescriptor <T>()) ?? new SearchDescriptor <T>();
resolvedDescriptor = resolvedDescriptor.Type(typeof(T).FullName);
return(tempClient.Search <T>(resolvedDescriptor));
}
/// <summary>
/// Provides quick search, filtered by current user
/// </summary>
/// <param name="client"></param>
/// <param name="query">Nest Query Container</param>
/// <param name="page"></param>
/// <param name="pageSize"></param>
/// <param name="searchRoots"></param>
/// <param name="includeTypes"></param>
/// <param name="excludeTypes"></param>
/// <param name="buildSearchHit">Can be used to customize how VulcanSearchHit is populated. Default is IVulcanClientExtensions.DefaultBuildSearchHit</param>
/// <param name="contentLoader"></param>
/// <returns></returns>
public static VulcanSearchHitList GetSearchHits(this IVulcanClient client,
QueryContainer query,
int page,
int pageSize,
IEnumerable <ContentReference> searchRoots = null,
IEnumerable <Type> includeTypes = null,
IEnumerable <Type> excludeTypes = null,
Func <IHit <IContent>, IContentLoader, VulcanSearchHit> buildSearchHit = null,
IContentLoader contentLoader = null
)
{
if (includeTypes == null)
{
var pageTypes = typeof(PageData).GetSearchTypesFor(x => x.IsClass && !x.IsAbstract);
var mediaTypes = typeof(MediaData).GetSearchTypesFor(x => x.IsClass && !x.IsAbstract);
includeTypes = pageTypes.Union(mediaTypes);
}
// restrict to start page and global blocks if not otherwise specified
if (searchRoots == null && !ContentReference.IsNullOrEmpty(ContentReference.StartPage))
{
searchRoots = new[] { ContentReference.StartPage, ContentReference.GlobalBlockFolder }
}
;
buildSearchHit = buildSearchHit ?? DefaultBuildSearchHit;
pageSize = pageSize < 1 ? 10 : pageSize;
page = page < 1 ? 1 : page;
var searchForTypes = includeTypes.Except(excludeTypes ?? new Type[] { });
var hits = client.SearchContent <VulcanContentHit>(d => d
.Skip((page - 1) * pageSize)
.Take(pageSize)
.FielddataFields(fs => fs.Field(SearchDescriptionField).Field(p => p.ContentLink)) // only return contentLink
.Query(q => query)
//.Highlight(h => h.Encoder("html").Fields(f => f.Field("*")))
.Aggregations(agg => agg.Terms("types", t => t.Field(TypeField))),
typeFilter: searchForTypes,
principleReadFilter: UserExtensions.GetUser(),
rootReferences: searchRoots,
includeNeutralLanguage: true
);
var searchHits = hits.Hits.Select(x => buildSearchHit(x, contentLoader ?? ContentLoader.Service));
var results = new VulcanSearchHitList(searchHits)
{
TotalHits = hits.Total, ResponseContext = hits, Page = page, PageSize = pageSize
};
return(results);
}
}
/// <summary>
/// Allows for creation/updates of index templates
/// </summary>
/// <param name="client"></param>
/// <param name="indexPrefix"></param>
/// <param name="logger"></param>
public static void RunCustomIndexTemplates(this IVulcanClient client, string indexPrefix, ILogger logger)
{
foreach (var customizer in Customizers)
{
try
{
var updateIndexTemplate = customizer?.CustomIndexTemplate?.Invoke(client, indexPrefix);
if (updateIndexTemplate?.IsValid == false)
{
logger.Error($"Could not update index template {client.IndexName}: {updateIndexTemplate.DebugInformation}");
}
}
catch (NotImplementedException) { }
}
}
/// <summary>
/// Allows customization on mappings
/// </summary>
/// <param name="client"></param>
/// <param name="logger"></param>
public static void RunCustomMappers(this IVulcanClient client, ILogger logger)
{
// run the mappings
foreach (var customizer in Customizers)
{
try
{
var mappingResponse = customizer?.CustomMapper?.Invoke(client);
if (mappingResponse?.IsValid == false)
{
logger.Error($"Could not add mapping for index {client.IndexName}: {mappingResponse.DebugInformation}");
}
}
catch (NotImplementedException) { }
}
}
/// <summary>
/// Initialize analyzer on elasticsearch
/// </summary>
/// <param name="client"></param>
protected virtual void InitializeAnalyzer(IVulcanClient client)
{
var language = VulcanHelper.GetAnalyzer(client.Language);
IUpdateIndexSettingsResponse response;
if (language != "standard")
{
// first, stop words
response = client.UpdateIndexSettings(client.IndexName, uix => uix
.IndexSettings(ixs => ixs
.Analysis(ana => ana
.TokenFilters(tf => tf
.Stop("stop", sw => sw
.StopWords(GetStopwordsLanguage(language)))))));
if (!response.IsValid)
{
Logger.Error($"Could not set up stop words for {client.IndexName}: {response.DebugInformation}");
}
// next, stemmer
if (!new[] { "cjk", "persian", "thai" }.Contains(language))
{
response = client.UpdateIndexSettings(client.IndexName, uix => uix
.IndexSettings(ixs => ixs
.Analysis(ana => ana
.TokenFilters(tf => tf
.Stemmer("stemmer", stm => stm
.Language(GetStemmerLanguage(language)))))));
if (!response.IsValid)
{
Logger.Error($"Could not set up stemmers for {client.IndexName}: {response.DebugInformation}");
}
}
// next, stemmer overrides
if (language == "dutch")
{
response = client.UpdateIndexSettings(client.IndexName, uix => uix
.IndexSettings(ixs => ixs
.Analysis(ana => ana
.TokenFilters(tf => tf
.StemmerOverride("override", stm => stm
.Rules("fiets=>fiets",
"bromfiets=>bromfiets",
"ei=>eier",
"kind=>kinder"))))));
if (!response.IsValid)
{
Logger.Error($"Could not set up stemmer overrides for {client.IndexName}: {response.DebugInformation}");
}
}
// next, elision
if (new[] { "catalan", "french", "irish", "italian" }.Contains(language))
{
response = client.UpdateIndexSettings(client.IndexName, uix => uix
.IndexSettings(ixs => ixs
.Analysis(ana => ana
.TokenFilters(tf => tf
.Elision("elision", e => e
.Articles(GetElisionArticles(language)))))));
if (!response.IsValid)
{
Logger.Error($"Could not set up elisions for {client.IndexName}: {response.DebugInformation}");
}
}
// next, possessive
if (language == "english")
{
response = client.UpdateIndexSettings(client.IndexName, uix => uix
.IndexSettings(ixs => ixs
.Analysis(ana => ana
.TokenFilters(tf => tf
.Stemmer("possessive", stm => stm
.Language("possessive_english"))))));
if (!response.IsValid)
{
Logger.Error($"Could not set up possessives for {client.IndexName}: {response.DebugInformation}");
}
}
// next, lowercase
if (new[] { "greek", "irish", "turkish" }.Contains(language))
{
response = client.UpdateIndexSettings(client.IndexName, uix => uix
.IndexSettings(ixs => ixs
.Analysis(ana => ana
.TokenFilters(tf => tf
.Lowercase("custom_lowercase", stm => stm
.Language(language))))));
if (!response.IsValid)
{
Logger.Error("Could not set up lowercases for " + client.IndexName + ": " + response.DebugInformation);
}
}
}
response = client.UpdateIndexSettings(client.IndexName, uix => uix
.IndexSettings(ixs => ixs
.Analysis(ana => ana
.TokenFilters(tf => tf
.Synonym("synonyms", syn => syn
.Synonyms(GetSynonyms(client))))
.Analyzers(a => a
.Custom("default", cad => cad
.Tokenizer("standard")
.Filters(GetFilters(language)))))));
if (!response.IsValid)
{
Logger.Error($"Could not set up custom analyzers for {client.IndexName}: {response.DebugInformation}");
}
if (language != "persian")
{
return;
}
response = client.UpdateIndexSettings(client.IndexName, uix => uix
.IndexSettings(ixs => ixs
.Analysis(ana => ana
.CharFilters(cf => cf
.Mapping("zero_width_spaces", stm => stm
.Mappings("\\u200C=> ")))
.Analyzers(a => a
.Custom("default", cad => cad
.CharFilters("zero_width_spaces"))))));
if (!response.IsValid)
{
Logger.Error("Could not set up char filters for " + client.IndexName + ": " + response.DebugInformation);
}
}
/// <summary>
/// Injected constructor
/// </summary>
/// <param name="vulcanHandler"></param>
public VulcanPocoIndexingJob(IVulcanHandler vulcanHandler)
{
_VulcanHander = vulcanHandler;
_InvariantClient = _VulcanHander.GetClient(CultureInfo.InvariantCulture);
}
void IVulcanPipelineInstaller.Install(IVulcanClient client)
{
if (!_vulcanAttachmentIndexerSettings.EnableAttachmentPlugins || !client.Language.Equals(CultureInfo.InvariantCulture))
{
return;
}
var info = client.NodesInfo();
if (info?.Nodes?.Any(x => x.Value?.Plugins?.Any(y => string.Compare(y.Name, PluginName, StringComparison.OrdinalIgnoreCase) == 0) == true) != true)
{
throw new Exception($"No attachment plugin found, be sure to install the '{PluginName}' plugin on your Elastic Search Server!");
}
#if NEST2
// v2, to do, get all MediaData types that are allowed and loop them
var mediaDataTypes = Core.Extensions.TypeExtensions.GetSearchTypesFor <MediaData>(t => t.IsAbstract == false);
foreach (var mediaType in mediaDataTypes)
{
var descriptors = mediaType.GetCustomAttributes(false).OfType <MediaDescriptorAttribute>();
var extensionStrings = string.Join(",", descriptors.Select(x => x.ExtensionString ?? ""));
var extensions = extensionStrings.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
var manualCheck = MapAttachment?.Invoke(mediaType) ?? false;
// only map ones we allow
if (!extensions.Intersect(_vulcanAttachmentIndexerSettings.SupportedFileExtensions).Any() && !manualCheck)
{
continue;
}
var response = client.Map <object>(m => m.
Index(client.IndexName). // was _all
Type(mediaType.FullName).
Properties(props => props.
Attachment(s => s.Name(MediaContents)
.FileField(ff => ff.Name("content").Store().TermVector(Nest.TermVectorOption.WithPositionsOffsets))
))
);
if (!response.IsValid)
{
throw new Exception(response.DebugInformation);
}
}
#elif NEST5
// v5, use pipeline
var response = client.PutPipeline(PipelineId, p => p
.Description("Document attachment pipeline")
.Processors(pr => pr
.Attachment <Nest.Attachment>(a => a
.Field(MediaContents)
.TargetField(MediaContents)
.IndexedCharacters(-1)
)
)
);
if (!response.IsValid)
{
throw new Exception(response.DebugInformation);
}
#endif
}