public string Compile(QuerySetup query, Facet facet, string facetCode)
{
string clauses = String.Join("", facet.Clauses.Select(x => x.Clause));
string sql = $@"
SELECT '{facetCode}' AS facet_code, MIN({facet.CategoryIdExpr}::real) AS min, MAX({facet.CategoryIdExpr}::real) AS max
FROM {query.Facet.TargetTable.ResolvedSqlJoinName}
{query.Joins.Combine("")}
{"WHERE ".GlueTo(clauses)}";
return(sql);
}
private static void SetupAttachmentFields(QuerySetup querySetup)
{
querySetup.SearchFields.Remove(DefaultFields.AttachmentData);
querySetup.SearchFields.Remove(DefaultFields.AttachmentContent);
querySetup.SearchFields.Remove(DefaultFields.AttachmentAuthor);
querySetup.SearchFields.Remove(DefaultFields.AttachmentKeywords);
querySetup.SearchFields.Add(DefaultFields.AttachmentContent);
querySetup.SearchFields.Add(DefaultFields.AttachmentAuthor);
querySetup.SearchFields.Add(DefaultFields.AttachmentKeywords);
}
public void Search_FromOver10k_Throws()
{
var setup = new QuerySetup
{
From = 10001,
SearchText = GetString(),
Language = CultureInfo.CurrentCulture
};
Assert.Throws <ArgumentOutOfRangeException>(() => _builder.Search(setup));
}
/// <summary>
/// Mocks IQuerySetupBuilder.Setup. Returns passed argument.
/// </summary>
/// <param name="querySetup"></param>
/// <returns></returns>
public virtual Mock <IQuerySetupBuilder> MockQuerySetupBuilder(QuerySetup querySetup)
{
var mockQuerySetupBuilder = new Mock <IQuerySetupBuilder>();
mockQuerySetupBuilder.Setup(x => x.Build(
It.IsAny <FacetsConfig2>(),
It.IsAny <Facet>(),
It.IsAny <List <string> >(),
It.IsAny <List <string> >()
)).Returns(querySetup ?? new QuerySetup {
});
return(mockQuerySetupBuilder);
}
public QueryRequest(QuerySetup querySetup)
{
querySetup.SearchText.EnsureNotNull(nameof(querySetup.SearchText));
Query.SearchText = querySetup.SearchText.ToLower();
_sortFields = querySetup.SortFields;
_scriptField = querySetup.ServerVersion >= Core.Constants.InlineVsSourceVersion
? JsonNames.ScriptSource
: JsonNames.Inline;
From = querySetup.From;
Size = querySetup.Size;
Operator = querySetup.Operator;
}
public void Filter_ReturnsExpectedJsonForDouble(double value)
{
var setup = new QuerySetup {
SearchText = "term", Language = _language
};
setup.Filters.Add(new Filter("MyField", value, typeof(double), false, Operator.And));
var query = (QueryRequest)_builder.TypedSearch <string>(setup);
var result = Serialize(query.PostFilter.Bool.Must);
var expected = GetJsonTestData($"PostFilterShouldDouble_{value}.json");
Assert.Equal(expected, result, ignoreLineEndingDifferences: true);
}
public void Search_IVersionable_WithApplyDefaultFilters_AddsPublishFilters()
{
var setup = new QuerySetup
{
SearchText = GetString(),
Type = typeof(IVersionable),
Language = CultureInfo.CurrentCulture,
ApplyDefaultFilters = true
};
var request = (QueryRequest)_builder.Search(setup);
var filters = request.Query.Bool.Filter.OfType <Range <DateTime> >();
Assert.Contains(filters, x => x.RangeSetting.Field == nameof(IVersionable.StopPublish));
Assert.Contains(filters, x => x.RangeSetting.Field == nameof(IVersionable.StartPublish));
}
public void FacetFieldNames_AddsAggregation(string field, MappingType type, string expectedKey)
{
var setup = new QuerySetup
{
SearchText = "term",
Language = _language,
FacetFieldNames = new Dictionary <string, MappingType>
{
{ field, type }
}
};
var request = (QueryRequest)_builder.TypedSearch <string>(setup);
var aggregation = request.Aggregation.First();
Assert.Equal(expectedKey, aggregation.Key);
}
public void FilterACL_AddsBoolShoulds()
{
var setup = new QuerySetup
{
SearchText = "term",
Language = _language,
AclPrincipal = GetPrincipalInfo("foo", "Role1", "Role2"),
AppendAclFilters = true
};
var request = (QueryRequest)_builder.TypedSearch <string>(setup);
var boolQuery = request.Query.Bool.Filter.Cast <NestedBoolQuery>().First().Bool;
var shoulds = boolQuery.Should.Cast <MatchSimple>();
Assert.Contains(shoulds, f => f.Match.Value <string>("_acl") == "U:foo");
Assert.Contains(shoulds, f => f.Match.Value <string>("_acl") == "R:Role1");
Assert.Contains(shoulds, f => f.Match.Value <string>("_acl") == "R:Role2");
}
public void Search_NoScriptScore_ReturnsExpectedJson()
{
const string expected1 = "function_score";
const string expected2 = "\"script_score\":{";
var builder = new QueryBuilder(null, null, null);
builder.SetMappedFields(new[] { "bar" });
var querySetup = new QuerySetup
{
SearchText = GetString(),
Language = _language
};
var result = RemoveWhitespace(Serialize(builder.Search(querySetup)));
Assert.DoesNotContain(expected1, result);
Assert.DoesNotContain(expected2, result);
}
private static void SetupAttachmentFields(QuerySetup querySetup)
{
if (!querySetup.SearchFields.Contains(DefaultFields.Attachment))
{
return;
}
querySetup.SearchFields.Remove(DefaultFields.Attachment);
querySetup.SearchFields.Remove(DefaultFields.AttachmentContent);
querySetup.SearchFields.Remove(DefaultFields.AttachmentAuthor);
querySetup.SearchFields.Remove(DefaultFields.AttachmentTitle);
querySetup.SearchFields.Remove(DefaultFields.AttachmentName);
querySetup.SearchFields.Remove(DefaultFields.AttachmentKeywords);
querySetup.SearchFields.Add(DefaultFields.AttachmentContent);
querySetup.SearchFields.Add(DefaultFields.AttachmentAuthor);
querySetup.SearchFields.Add(DefaultFields.AttachmentTitle);
querySetup.SearchFields.Add(DefaultFields.AttachmentName);
querySetup.SearchFields.Add(DefaultFields.AttachmentKeywords);
}
public void Search_GaussAndScriptScore_Throws()
{
var setup = new QuerySetup
{
SearchText = GetString(),
ScriptScore = new ScriptScore
{
Script = new ScriptScore.ScriptScoreInner
{
Language = "painless",
Source = GetString()
}
}
};
setup.Gauss.Add(new Gauss());
Exception exception = Assert.Throws <InvalidOperationException>(() => _builder.Search(setup));
Assert.Equal("Cannot use Gauss and ScriptScore simultaneously", exception.Message);
}
public void Build_WithConcretePickCompilerAndVariousConfigs_GivesExpecteCriteriadCount(string uri)
{
// Arrange
var facetsConfig = FakeFacetsConfig(uri);
var pickCriterias = new List <string> {
"Q1 = Q2", "Q3 = Q4"
};
var mockPicksCompiler = MockPicksFilterCompiler(pickCriterias ?? new List <string>());
var fakeJoins = FakeJoinsClause(5);
var mockJoinCompiler = MockJoinsClauseCompiler(fakeJoins);
var mockRoutes = new List <GraphRoute> {
FakeRoute2("A", "B", "C", "D"),
FakeRoute2("E", "K")
};
var mockFacetsGraph = MockFacetsGraph(mockRoutes);
var extraTables = new List <string>();
// Act
var builder = new QuerySetupBuilder(mockFacetsGraph.Object, mockPicksCompiler.Object, mockJoinCompiler.Object);
QuerySetup querySetup = builder.Build(facetsConfig, facetsConfig.TargetFacet, extraTables, null);
Output.WriteLine($"URI: {uri}");
foreach (var criteria in querySetup.Criterias)
{
Output.WriteLine($" criteria: {criteria}");
}
//DumpUriObject(uri, querySetup);
// Assert
Assert.Same(facetsConfig.TargetConfig, querySetup.TargetConfig);
Assert.Same(facetsConfig.TargetFacet, querySetup.Facet);
// Assert.Equal(mockRoutes.Aggregate(0, (i,z) => i + z.Items.Count), querySetup.Joins.Count);
}
/// <summary>
/// A standard freetext search against all fields, constrained by type T. With facets.
/// </summary>
/// <param name="querySetup">The QuerySetup. <see cref="QuerySetup"/></param>
public RequestBase TypedSearch <T>(QuerySetup querySetup) where T : class
{
querySetup.Type = typeof(T);
return(TypedSearch(querySetup));
}
private static void SetupFilters(QuerySetup setup, QueryRequest request)
{
var filterQuery = new NestedBoolQuery();
// Filter away excluded types
if (setup.ExcludedTypes.Count > 0)
{
filterQuery.Bool.MustNot.AddRange(setup.ExcludedTypes.Select(e => new MatchSimple(DefaultFields.Types, e.GetTypeName().ToLower())));
}
foreach (var ex in GetExcludedRoots(setup))
{
filterQuery.Bool.MustNot.Add(new MatchSimple(DefaultFields.Id, ex.Key.ToString()));
if (ex.Value)
{
filterQuery.Bool.MustNot.Add(new MatchSimple(DefaultFields.Path, ex.Key.ToString()));
}
}
// Filter on type
if (setup.Type != null)
{
var term = CreateTerm(new Filter(DefaultFields.Types, setup.Type.GetTypeName().ToLower(), null, false, Operator.And));
filterQuery.Bool.Must.Add(term);
}
// Filter on ranges
if (setup.Ranges.Count > 0)
{
request.Query.Bool.Filter.AddRange(setup.Ranges);
}
// Filter on root-id
if (setup.RootId != 0)
{
var term = new Term(DefaultFields.Path, Convert.ToString(setup.RootId), true);
filterQuery.Bool.Must.Add(term);
}
// Filter on ACL
if (setup.AppendAclFilters && setup.AclPrincipal != null)
{
var boolQuery = new NestedBoolQuery();
foreach (var role in setup.AclPrincipal.RoleList)
{
var roleTerm = new MatchSimple(DefaultFields.Acl, $"R:{role}");
boolQuery.Bool.Should.Add(roleTerm);
}
var userTerm = new MatchSimple(DefaultFields.Acl, $"U:{setup.AclPrincipal.Name}");
boolQuery.Bool.Should.Add(userTerm);
boolQuery.Bool.MinimumNumberShouldMatch = 1;
request.Query.Bool.Filter.Add(boolQuery);
}
if (setup.FilterGroups.Count > 0)
{
foreach (var filterGroup in setup.FilterGroups)
{
foreach (var filter in filterGroup.Value.Filters)
{
var boolQuery = new BoolQuery();
if (filter.Operator == Operator.Or)
{
boolQuery.MinimumNumberShouldMatch = 1;
boolQuery.Should = new List <MatchBase>();
if (ArrayHelper.IsArrayCandidate(filter.Value.GetType()))
{
foreach (object value in (IEnumerable)ArrayHelper.ToArray(filter.Value))
{
boolQuery.Should.Add(Term.FromFilter(filter, value));
}
}
else
{
boolQuery.Should.Add(Term.FromFilter(filter));
}
}
else if (filter.Operator == Operator.And)
{
boolQuery.Must = new List <MatchBase>();
if (ArrayHelper.IsArrayCandidate(filter.Value.GetType()))
{
foreach (object value in (IEnumerable)ArrayHelper.ToArray(filter.Value))
{
boolQuery.Must.Add(Term.FromFilter(filter, value));
}
}
else
{
boolQuery.Must.Add(Term.FromFilter(filter));
}
}
// Use regular or post filter?
if (!setup.UsePostfilters)
{
request.Query.Bool.Filter.Add(Term.FromArrayFilter(filter));
}
else
{
if (filterGroup.Value.Operator == Operator.And)
{
request.PostFilter.Bool.Must.Add(new NestedBoolQuery(boolQuery));
}
else if (filterGroup.Value.Operator == Operator.Or)
{
request.PostFilter.Bool.Should.Add(new NestedBoolQuery(boolQuery));
}
}
}
}
}
if (setup.Filters.Count > 0)
{
// Add not-filters as regular filter regardless of post-filter value
IEnumerable <Filter> notFilters = setup.Filters.Where(f => f.Not).ToArray();
filterQuery.Bool.MustNot.AddRange(notFilters.Select(CreateTerm));
// Use regular or post filter?
if (!setup.UsePostfilters)
{
request.Query.Bool.Filter.AddRange(
setup.Filters
.Except(notFilters)
.Select(CreateTerm));
}
else
{
request.PostFilter.Bool.Must.AddRange(
setup.Filters
.Except(notFilters)
.Where(f => f.Operator == Operator.And && !f.Not)
.Select(CreateTerm));
request.PostFilter.Bool.Should.AddRange(
setup.Filters
.Except(notFilters)
.Where(f => f.Operator == Operator.Or && !f.Not)
.Select(CreateTerm));
}
}
if (filterQuery.HasAnyValues())
{
request.Query.Bool.Filter.Add(filterQuery);
}
AppendDefaultFilters(request.Query, setup.Type);
if (request.Query.Bool.Should.Count > 1 && request.Query.Bool.Must.Count == 0)
{
request.Query.Bool.MinimumNumberShouldMatch = 1;
}
else
{
request.Query.Bool.MinimumNumberShouldMatch = null;
}
if (request.PostFilter.Bool.Should.Count > 0 && request.PostFilter.Bool.Must.Count == 0)
{
request.PostFilter.Bool.MinimumNumberShouldMatch = 1;
}
else
{
request.PostFilter.Bool.MinimumNumberShouldMatch = null;
}
}
private void SetupBoosting(QuerySetup setup, QueryRequest request)
{
if (!setup.UseBoosting)
{
return;
}
List <Boost> boosting = GetBoosting(setup.Type, setup.BoostFields);
if (boosting.Count > 0)
{
var searchText = request.Query.SearchText.Replace("*", String.Empty);
if (!TextUtil.IsNumeric(searchText))
{
boosting.RemoveAll(b => b.FieldName.Equals(DefaultFields.Id));
}
request.Query.Bool.Should.AddRange(
boosting.Select(b =>
new MatchWithBoost(b.FieldName,
searchText, b.Weight, setup.Operator)));
}
// Boosting by type
if (setup.BoostTypes.Count > 0)
{
request.Query.Bool.Should.AddRange(
setup.BoostTypes.Select(b =>
new MatchWithBoost(DefaultFields.Types, b.Key.GetTypeName(), b.Value, setup.Operator)));
// Direct match in Type gives higher score than match in Types, hence the +1
request.Query.Bool.Should.AddRange(
setup.BoostTypes.Select(b =>
new MatchWithBoost(DefaultFields.Type, b.Key.GetTypeName(), b.Value + 1, setup.Operator)));
}
if (setup.BoostAncestors.Count > 0)
{
request.Query.Bool.Should.AddRange(
setup.BoostAncestors.Select(b =>
new MatchWithBoost(DefaultFields.Path, b.Key.ToString(), b.Value, setup.Operator)));
request.Query.Bool.Should.AddRange(
setup.BoostAncestors.Select(b =>
new MatchWithBoost(DefaultFields.Id, b.Key.ToString(), b.Value, setup.Operator)));
}
// Best Bets
if (setup.UseBestBets && !String.IsNullOrWhiteSpace(request.Query.SearchText))
{
IEnumerable <string> terms = request.Query.SearchText
.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)
.Select(t => t.Trim().Trim('*'));
var key = setup.IndexName ?? _settings.GetDefaultIndexName(Language.GetLanguageCode(setup.Language));
if (!Conventions.Indexing.BestBets.TryGetValue(key, out var bestBetsForLanguage))
{
return;
}
IEnumerable <BestBet> bestBets = bestBetsForLanguage
.Where(b => b.Terms.Any(t => terms.Contains(t)));
request.Query.Bool.Should.AddRange(
bestBets.Select(_ =>
new MatchWithBoost(
DefaultFields.BestBets, request.Query.SearchText.Trim('*'), BestBetMultiplier, setup.Operator)));
}
}
private RequestBase SearchInternal(QuerySetup setup)
{
_searchType = setup.SearchType;
setup.SearchText.EnsureNotNull("searchText");
if (setup.From > 10000 || setup.Size > 10000)
{
throw new ArgumentOutOfRangeException(nameof(setup),
"From (skip) and size (take) must be less than or equal to: 10000. If you really must, this limit can be set by changing the [index.max_result_window] index level parameter");
}
var request = new QueryRequest(setup);
request.Query.SearchText = setup.SearchText.ToLower();
if (setup.SearchFields.Count == 0)
{
setup.SearchFields.AddRange(GetMappedFields(Language.GetLanguageCode(setup.Language), setup.IndexName, setup.SearchType));
}
if (Log.IsDebugEnabled())
{
Log.Debug("SearchFields:");
setup.SearchFields.ForEach(f => Log.Debug(f));
}
SetupAttachmentFields(setup);
SetupSourceFields(request, setup);
if (setup.IsGetQuery)
{
request.Query.Bool.Must.Add(new MatchAll());
}
else if (setup.IsWildcard)
{
setup.SearchFields.ForEach(field =>
request.Query.Bool.Should.Add(new Wildcard(field, request.Query.SearchText)));
// Boost hits that starts with searchtext, ie. when searching for "*foo*",
// hits on "foobar" will score higher than hits on "barfoo"
if (request.Query.SearchText.StartsWith("*"))
{
setup.SearchFields.ForEach(field =>
request.Query.Bool.Should.Add(new Wildcard(field, request.Query.SearchText.TrimStart('*'), 10)));
}
request.Query.Bool.MinimumNumberShouldMatch = 1;
}
else
{
request.Query.Bool.Must.Add(
new MatchMulti(
request.Query.SearchText,
setup.SearchFields,
setup.Operator,
null,
null,
setup.FuzzyLength,
setup.Analyzer));
// Boost phrase matches if multiple words
if (request.Query.SearchText?.IndexOf(" ", StringComparison.OrdinalIgnoreCase) > 0)
{
request.Query.Bool.Should.Add(
new MatchMulti(
request.Query.SearchText,
setup.SearchFields,
setup.Operator,
"phrase",
2));
}
}
SetupBoosting(setup, request);
if (setup.FacetFieldNames.Count > 0)
{
request.Aggregation = GetAggregationQuery(setup.FacetFieldNames);
}
SetupFilters(setup, request);
// Highlighting
if (setup.UseHighlight)
{
request.Highlight = new Highlight
{
Fields = GetHighlightFields()
};
}
// Did-you-mean
if (setup.EnableDidYouMean)
{
request.DidYouMeanSuggest = new DidYouMeanSuggest(request.Query.SearchText);
}
// function_score. Must be the last operation in this method.
// Cannot use Gauss and ScriptScore simultaneously
if (setup.ScriptScore != null && setup.Gauss.Count > 0)
{
throw new Exception("Cannot use Gauss and ScriptScore simultaneously");
}
if (setup.ScriptScore != null)
{
request.Query = new FunctionScoreQuery(request.Query, setup.ScriptScore);
}
if (setup.Gauss.Count > 0)
{
request.Query = new FunctionScoreQuery(request.Query, setup.Gauss);
}
return(request);
}
internal RequestBase MoreLikeThis(QuerySetup setup)
{
return(new MoreLikeThisRequest(setup));
}
internal RequestBase MoreLikeThis(QuerySetup setup) => new MoreLikeThisRequest(setup);
/// <summary>
/// See https://www.elastic.co/guide/en/elasticsearch/reference/current/search-suggesters-completion.html
/// </summary>
public SuggestRequest Suggest(QuerySetup querySetup)
{
return(new SuggestRequest(querySetup.SearchText, querySetup.Size));
}
/// <summary>
/// A standard freetext search against all fields
/// </summary>
/// <param name="querySetup">The QuerySetup. <see cref="QuerySetup"/></param>
public RequestBase Search(QuerySetup querySetup)
{
return(SearchInternal(querySetup));
}
/// <summary> /// A standard freetext search against all fields /// </summary> /// <param name="querySetup">The QuerySetup. <see cref="QuerySetup"/></param> public RequestBase Search(QuerySetup querySetup) => SearchInternal(querySetup);
private RequestBase SearchInternal(QuerySetup setup)
{
CheckSize(setup);
var request = new QueryRequest(setup);
if (setup.SearchFields.Count == 0)
{
setup.SearchFields.AddRange(GetMappedFields(Language.GetLanguageCode(setup.Language), setup.IndexName, setup.SearchType));
}
if (_logger.IsDebugEnabled())
{
_logger.Debug("SearchFields:");
setup.SearchFields.ForEach(f => _logger.Debug(f));
}
SetupAttachmentFields(setup);
SetupSourceFields(request, setup);
if (setup.IsGetQuery)
{
request.Query.Bool.Must.Add(new MatchAll());
}
else if (setup.IsWildcard)
{
setup.SearchFields.ForEach(field =>
request.Query.Bool.Should.Add(new Wildcard(field, request.Query.SearchText)));
// Boost hits that starts with searchtext, ie. when searching for "*foo*",
// hits on "foobar" will score higher than hits on "barfoo"
if (request.Query.SearchText.StartsWith("*"))
{
setup.SearchFields.ForEach(field =>
request.Query.Bool.Should.Add(new Wildcard(field, request.Query.SearchText.TrimStart('*'), 10)));
}
request.Query.Bool.MinimumNumberShouldMatch = 1;
}
else
{
request.Query.Bool.Must.Add(
new MatchMulti(
request.Query.SearchText,
setup.SearchFields,
setup.Operator,
null,
null,
setup.FuzzyLength,
setup.Analyzer));
// Boost phrase matches if multiple words
if (request.Query.SearchText?.IndexOf(" ", StringComparison.OrdinalIgnoreCase) > 0)
{
request.Query.Bool.Should.Add(
new MatchMulti(
request.Query.SearchText,
setup.SearchFields,
setup.Operator,
"phrase",
2));
}
}
SetupBoosting(setup, request);
if (setup.FacetFieldNames.Count > 0)
{
request.Aggregation = GetAggregationQuery(setup.FacetFieldNames);
}
SetupFilters(setup, request);
// Highlighting
if (setup.UseHighlight)
{
request.Highlight = new Highlight
{
Fields = GetHighlightFields()
};
}
// Did-you-mean
if (setup.EnableDidYouMean)
{
request.DidYouMeanSuggest = new DidYouMeanSuggest(request.Query.SearchText);
}
// function_score. Must be the last operation in this method.
// Cannot use Gauss and ScriptScore simultaneously
if (setup.ScriptScore != null && setup.Gauss.Count > 0)
{
throw new InvalidOperationException("Cannot use Gauss and ScriptScore simultaneously");
}
if (setup.ScriptScore != null)
{
request.Query = new FunctionScoreQuery(request.Query, setup.ScriptScore);
}
if (setup.Gauss.Count > 0)
{
request.Query = new FunctionScoreQuery(request.Query, setup.Gauss);
}
return(request);
}
