public static IAnalysis ProjectAnalysisSettings(AnalysisDescriptor analysis) { analysis .TokenFilters(tokenFilters => tokenFilters .Shingle("shingle", shingle => shingle .MinShingleSize(2) .MaxShingleSize(4) ) ) .Analyzers(analyzers => analyzers .Custom("shingle", shingle => shingle .Filters("standard", "shingle") .Tokenizer("standard") ) ); //normalizers are a new feature since 5.2.0 if (TestConfiguration.Instance.InRange(">=5.2.0")) { analysis.Normalizers(analyzers => analyzers .Custom("my_normalizer", n => n .Filters("lowercase", "asciifolding") ) ); } return(analysis); }
private IAnalysis ConfigureConcatenateAndAutocompleteAnalysis(AnalysisDescriptor analysis) { // for concatenate filter see my fork: https://github.com/rh78/elasticsearch-concatenate-token-filter return(analysis .TokenFilters(filter => filter .UserDefined("concatenate_filter", new ConcatenateTokenFilter() { TokenSeparator = " ", IncrementGap = 1000 }) .EdgeNGram("edge_ngram_filter", edgeNGram => edgeNGram .MinGram(1) .MaxGram(50) ) .NGram("ngram_filter", nGram => nGram .MinGram(3) .MaxGram(50) ) .NGram("short_ngram_filter", nGram => nGram .MinGram(1) .MaxGram(50) ) ) .Analyzers(analyzer => analyzer .Custom("edge_ngram_concatenate_index", custom => custom .Tokenizer("standard") .Filters(new string[] { "lowercase", "asciifolding", "concatenate_filter", "edge_ngram_filter" }) ) .Custom("edge_ngram_partial_index", custom => custom .Tokenizer("standard") .Filters(new string[] { "lowercase", "asciifolding", "edge_ngram_filter" }) ) .Custom("ngram_concatenate_index", custom => custom .Tokenizer("standard") .Filters(new string[] { "lowercase", "asciifolding", "concatenate_filter", "ngram_filter" }) ) .Custom("short_ngram_concatenate_index", custom => custom .Tokenizer("standard") .Filters(new string[] { "lowercase", "asciifolding", "concatenate_filter", "short_ngram_filter" }) ) .Custom("ngram_partial_index", custom => custom .Tokenizer("standard") .Filters(new string[] { "lowercase", "asciifolding", "ngram_filter" }) ) .Custom("short_ngram_partial_index", custom => custom .Tokenizer("standard") .Filters(new string[] { "lowercase", "asciifolding", "short_ngram_filter" }) ) .Custom("concatenate_search", custom => custom .Tokenizer("standard") .Filters(new string[] { "lowercase", "asciifolding", "concatenate_filter" }) ) .Custom("partial_search", custom => custom .Tokenizer("standard") .Filters(new string[] { "lowercase", "asciifolding" }) ) )); }
private static AnalysisDescriptor CreateAnalysisDescriptor(string charHtmlFilter, TokenFiltersDescriptor tokenFilters, AnalyzersDescriptor analyzers) { var analysisDescriptor = new AnalysisDescriptor(); analysisDescriptor.CharFilters(c => c.HtmlStrip(charHtmlFilter)); analysisDescriptor.TokenFilters(t => tokenFilters); analysisDescriptor.Analyzers(a => analyzers); return(analysisDescriptor); }
public IAnalysis ConfigureAnalysis(AnalysisDescriptor analysisDescriptor) { return(analysisDescriptor.TokenFilters(tf => tf.Synonym("city_synonym", tfd => tfd.Synonyms("lol => laughing", "new york, nyc"))) .Analyzers(aa => aa.Custom("cna", ca => ca .CharFilters("html_strip") .Tokenizer("standard") .Filters("lowercase", "stop", "city_synonym")) )); }
protected override AnalysisDescriptor Contribute(AnalysisDescriptor descriptor, IEnumerable <KeyValuePair <string, TokenFilterBase> > build) { return(descriptor.TokenFilters(a => { foreach (var item in build.Where(x => CanContribute(x, a))) { a.Add(item.Key, item.Value); } return a; })); }
private IAnalysis ProjectAnalysisSettings(AnalysisDescriptor analysis) => analysis .TokenFilters(tokenFilters => tokenFilters .Shingle("shingle", shingle => shingle .MinShingleSize(2) .MaxShingleSize(4) ) ) .Analyzers(analyzers => analyzers .Custom("shingle", shingle => shingle .Filters("standard", "shingle") .Tokenizer("standard") ) );
protected override IAnalysis FluentAnalysis(AnalysisDescriptor an) => an.TokenFilters(d => AssertionSetup.Fluent(AssertionSetup.Name, d));
public static AnalysisDescriptor DutchAnalysis(AnalysisDescriptor analysis) => analysis // custom filters .TokenFilters(tokenfilters => tokenfilters .Stop("dutch_stop", w => w .StopWords("_dutch_") ) .Stemmer("dutch_stemmer", w => w .Language("dutch") ) ) .CharFilters(charFilters => charFilters .PatternReplace("kill_numbers", p => p .Pattern("(\\d+)") .Replacement(""))) // custom analyzers .Analyzers(analyzers => analyzers .Custom("dutch", c => c .CharFilters("kill_numbers") .Tokenizer("standard") .Filters("lowercase", "dutch_stop", "dutch_stemmer") ) );
private IAnalysis CreateAnalysis(AnalysisDescriptor analysisDescriptor) { return(analysisDescriptor .TokenFilters(CreateTokenFilters) .Analyzers(CreateAnalyzers)); }
/// <summary> /// Job Analysis descriptions /// </summary> /// private AnalysisDescriptor GetJobsAnalysisDescriptor(LanguageCode languageCode = LanguageCode.EN) { var descriptor = new AnalysisDescriptor(); descriptor.TokenFilters(cf => cf.Add("shingle_title", new ShingleTokenFilter())); descriptor.TokenFilters( f => f.Add("job_stopfilter", new StopTokenFilter { Stopwords = new List <string> { "job", "jobs" } })); // Title Analyzer var titleAnalyzer = GetTitleAnalyzer(languageCode); descriptor.Analyzers(a => a.Add("job_title", titleAnalyzer)); // Path Analyzer var pathAnalyzer = GetPathAnalyzer(); descriptor.Analyzers(a => a.Add("path", pathAnalyzer)); // Lowercase Analyzer var lowercaseAnalyzer = GetLowercaseAnalyzer(languageCode); descriptor.Analyzers(a => a.Add("lowercase", lowercaseAnalyzer)); // Snowball Token Filter var snowballPorterFilter = GetStemmerTokenFilter(languageCode); descriptor.TokenFilters(d => d.Add("snowballPorterFilter", snowballPorterFilter)); // Stopwords Filter var stopwordFilter = GetStopwordFilter(languageCode); descriptor.TokenFilters(d => d.Add("stopwordFilter", stopwordFilter)); // Word Delimiter Token Filter var wdtFitler = GetWordDelimeterTokenFilter(languageCode); descriptor.TokenFilters(d => d.Add("wdtFitler", wdtFitler)); // Job Default Analyzer var jobDefaultAnalyzer = GetJobDefaultAnanyzer(languageCode); descriptor.Analyzers(a => a.Add("jobDefaultAnalyzer", jobDefaultAnalyzer)); // Job Default with Delimiter Analyzer var jobDefaultWithDelimiterAnalyzer = GetJobDefaultWithDelimiterAnalyzer(languageCode); descriptor.Analyzers(a => a.Add("jobDefaultWithDelimiterAnalyzer", jobDefaultWithDelimiterAnalyzer)); // Title Suggestion Anlyzer var titleSuggestAnalyzer = GetTitleSuggestAnalyzer(languageCode); descriptor.Analyzers(a => a.Add("titleSuggestAnalyzer", titleSuggestAnalyzer)); // country, match first node in hierarchy path descriptor.Tokenizers(t => t.Add("country_path", new PatternTokenizer { Pattern = "^(/[0-9]+/).*", Group = 1 })); descriptor.Analyzers(a => a.Add("country_path", new CustomAnalyzer { Tokenizer = "country_path" })); // region, match first and second nodes in hierarchy path descriptor.Tokenizers(t => t.Add("region_path", new PatternTokenizer { Pattern = "^(/[0-9]+/[0-9]+/).*", Group = 1 })); descriptor.Analyzers(a => a.Add("region_path", new CustomAnalyzer { Tokenizer = "region_path" })); // city, match first four or first three nodes in path as cities in some countries lack a second level division descriptor.Tokenizers(t => t.Add("city_path", new PatternTokenizer { Pattern = "^(/[0-9]+/[0-9]+/[0-9]+/[0-9]+/[0-9]+/[0-9]+/|/[0-9]+/[0-9]+/[0-9]+/[0-9]+/[0-9]+/|/[0-9]+/[0-9]+/[0-9]+/[0-9]+/|/[0-9]+/[0-9]+/[0-9]+/).*", Group = 1 })); descriptor.Analyzers(a => a.Add("city_path", new CustomAnalyzer { Tokenizer = "city_path" })); return(descriptor); }
private static IAnalysis SetupAnalysis(AnalysisDescriptor analysis) { return(analysis. TokenFilters(SetupTokenFilter) .Analyzers(SetupAnalyzer)); }