private static Func <QueryContainerDescriptor <TIndexItem>, QueryContainer> CreateQuery(ISearchArgs searchArgs,
Collection <TextSearchField <TIndexItem> > textSearchFields,
ICollection <Func <QueryContainerDescriptor <TIndexItem>, QueryContainer> > filters =
null)
{
var boolQueryDescriptor = new BoolQueryDescriptor <TIndexItem>();
if (!string.IsNullOrWhiteSpace(searchArgs.SearchText))
{
if (textSearchFields == null || !textSearchFields.Any())
{
throw new ElasticSearchException("At least one text field need to be specified for text search");
}
var textSearch = WildcardTextSearch(searchArgs.SearchText, textSearchFields);
boolQueryDescriptor.Must(textSearch);
}
filters ??= new Collection <Func <QueryContainerDescriptor <TIndexItem>, QueryContainer> >();
boolQueryDescriptor.Filter(filters);
QueryContainer Query(QueryContainerDescriptor <TIndexItem> q) => q.Bool(b => boolQueryDescriptor);
return(Query);
}
private SearchDescriptor <DocumentElastic> GetSampleDescriptor(string seed, IEnumerable <string> tagIds, string tagField)
{
var sdesc = new SearchDescriptor <DocumentElastic>();
var queryContDesc = new QueryContainerDescriptor <DocumentElastic>();
var queryContainers = new List <QueryContainer>();
if (tagIds != null && tagIds.Any())
{
var shouldDesc = new BoolQueryDescriptor <DocumentElastic>();
foreach (var batchTagIds in tagIds.Batch(1000))
{
shouldDesc.Should(queryContDesc
.Terms(t => t
.Terms(batchTagIds)
.Field(MapDocumentObjectName(tagField))));
}
queryContainers.Add(queryContDesc.Bool(q => shouldDesc));
}
queryContainers.Add(
queryContDesc.FunctionScore(f => f.Functions(fun => fun.RandomScore(seed)))
);
sdesc.Query(q => q.Bool(b => b.Must(queryContainers.ToArray())));
return(sdesc);
}
private BoolQueryDescriptor <FTPEntry> _mustHaveParents(BoolQueryDescriptor <FTPEntry> descriptor, bool HaveParents)
{
if (HaveParents)
{
return(descriptor.Must(m => m
.Exists(e => e
.Field(f => f.Parent)
)
));
}
else
{
return(descriptor.MustNot(mn => mn
.Exists(e => e
.Field(f => f.Parent)
)
));
}
}
public static BoolQueryDescriptor <T> Filter <T>(
this BoolQueryDescriptor <T> value,
object filter,
IDictionary <Type, Func <object, string> > filterValueFormatters = null)
where T : class
{
if (filter == null)
{
return(value);
}
var mustQueries = FilterLogic.GenerateMustQueriesFromFilter <T>(filter, filterValueFormatters);
if (mustQueries.Any())
{
return(value.Filter(mustQueries));
}
return(value);
}
private static BoolQueryDescriptor <DocumentDM> ExactQuery(DocumentDM doc)
{
var boolQuery = new BoolQueryDescriptor <DocumentDM>();
//ex
boolQuery.Must(mu => mu
.Match(m => !doc.Subject[0].Equals('?') ? m
.Field(f => f.Subject)
.Query(doc.Subject) : m
), mud => mud
.Match(m => !doc.Predicate[0].Equals('?') ? m
.Field(f => f.Predicate)
.Query(doc.Predicate) : m
),
mud => mud
.Match(m => !doc.Obj[0].Equals('?') ? m
.Field(f => f.Obj)
.Query(doc.Obj) : m
)
);
return(boolQuery);
}
/// <summary>
///
/// </summary>
/// <returns></returns>
public async Task <ISearchResponse <T> > ToListAsync()
{
SearchDescriptor <T> selector = new SearchDescriptor <T>();
BoolQueryDescriptor <T> boolQuery = new BoolQueryDescriptor <T>();
if (_mustSelector.Count > 0)
{
boolQuery.Must(_mustSelector);
}
if (_mustNotSelector.Count > 0)
{
boolQuery.MustNot(_mustNotSelector);
}
if (!string.IsNullOrWhiteSpace(IndexName))
{
selector.Index(IndexName);
}
selector = selector.Query(q => q.Bool(b => boolQuery)).From(SkipCount ?? 0);
if (TakeCount > 0)
{
selector.Size(TakeCount.Value);
}
if (sortor != null)
{
selector.Sort(sortor);
}
var response = await _searchContext.Context.SearchAsync <T>(selector);
if (!response.IsValid)
{
_searchContext.Logger.LogError($"[Success:{response.ApiCall.Success}]\t{response.ApiCall.Uri}");
_searchContext.Logger.LogError(response.ApiCall.DebugInformation);
_searchContext.Logger.LogError(response.ApiCall.OriginalException?.Message);
}
return(response);
}
#pragma warning restore IDE0039 // Use local function
static IBoolQuery TestBoolQuery(BoolQueryDescriptor <Person> b) => b.Name("thing");
private DataTable ElasticSearch(DocumentDM doc)
{
DataTable table = new DataTable();
ConnectionSettings connectionSettings;
ElasticClient elasticClient;
StaticConnectionPool connectionPool;
var nodes = new Uri[]
{
new Uri("http://localhost:9200/"),
};
connectionPool = new StaticConnectionPool(nodes);
connectionSettings = new ConnectionSettings(connectionPool);
elasticClient = new ElasticClient(connectionSettings);
//var searchResult = elasticClient.Search<DocumentDM>(s => s
// .Size(500)
// .Index("*")
// .Query(q => q
// .MultiMatch(m => m
// //get fields that are not variables
// .Fields(doc.GetTripleFields().Split(' ').ToList<string>().Select(x => new Field(x)).ToArray())
// //.Fields(f=> f
// // .Field("subject")
// // .Field("predicate")
// // .Field("object"))
// //get text that is not variables
// .Query(doc.GetTripleText())
// )
// )
// );
BoolQueryDescriptor <DocumentDM> boolQuery = ExactQuery(doc);
var searchResult = elasticClient.Search <DocumentDM>(s => s
.Size(500)
.Index("*")
.Query(q => q.Bool(b => boolQuery))
);
table.Columns.Add("subject", typeof(String));
table.Columns.Add("predicate", typeof(String));
table.Columns.Add("object", typeof(String));
table.Columns.Add("score", typeof(String));
table.Columns.Add("index", typeof(String));
table.Columns.Add("invisible", typeof(String));
for (int i = 0; i < searchResult.Documents.Count; i++)
{
var subject = searchResult.Documents.ElementAt(i).Subject;
var predicate = searchResult.Documents.ElementAt(i).Predicate;
var obj = searchResult.Documents.ElementAt(i).Obj;
var score = searchResult.HitsMetadata.Hits.ElementAt(i).Score;
var index = searchResult.HitsMetadata.Hits.ElementAt(i).Index;
StringBuilder sb = new StringBuilder();
sb.Append(subject);
sb.Append(" ");
sb.Append(predicate);
sb.Append(" ");
sb.Append(obj);
sb.Append(" ");
sb.Append(score);
sb.Append(" ");
sb.Append(index);
var row = new object[] { subject, predicate, obj, score, index, sb.ToString() };
table.Rows.Add(row);
}
return(table);
//if (!doc.Subject[0].Equals('?'))
//{
// var subjectResponse = elasticClient.Search<DocumentDM>(sd => sd
// .Index("*")
// .Size(1000)
// .Query(q => q
// .Match(m => m.Field("subject").Query(doc.Subject)
// )));
//}
//if (!doc.Predicate[0].Equals('?'))
//{
// var predicateResponse = elasticClient.Search<DocumentDM>(sd => sd
// .Index("*")
// .Size(1000)
// .Query(q => q
// .Match(m => m.Field("predicate").Query(doc.Predicate)
// )));
//}
//if (!doc.Obj[0].Equals('?'))
//{
// var objectResponse = elasticClient.Search<DocumentDM>(sd => sd
// .Index("*")
// .Size(1000)
// .Query(q => q
// .Match(m => m.Field("object").Query(doc.Obj)
// )));
//}
}
public ScrolledSearchResult <DocumentElastic> Filter(
string generalQuery,
IEnumerable <string> tagIds, string tagField,
int limit,
string orderBy, bool isDescending,
IEnumerable <string> interPretedFields,
IEnumerable <string> documentObjectFieldNames,
IEnumerable <string> returningDocumentObjectFields,
IEnumerable <string> ids = null,
DateTime?dateStart = null,
DateTime?dateEnd = null,
string shouldQuery = null)
{
var sdesc = new SearchDescriptor <DocumentElastic>();
var queryContDesc = new QueryContainerDescriptor <DocumentElastic>();
var queryContainers = new List <QueryContainer>();
if (!string.IsNullOrEmpty(generalQuery))
{
var modifiedQuery = generalQuery;
//replace the field names (because of the document_object)
if (documentObjectFieldNames?.Any() == true)
{
modifiedQuery = PrefixQueryFields(modifiedQuery, documentObjectFieldNames);
}
if (interPretedFields != null && interPretedFields.Any())
{
queryContainers.Add(
queryContDesc.QueryString(q => q
.Query(modifiedQuery)
.Fields(interPretedFields.Select(f => MapDocumentObjectName(f)).ToArray())));
}
else
{
queryContainers.Add(
queryContDesc.QueryString(q => q.Query(modifiedQuery)));
}
}
if (tagIds != null && tagIds.Any())
{
var shouldDesc = new BoolQueryDescriptor <DocumentElastic>();
foreach (var batchTagIds in tagIds.Batch(1000))
{
shouldDesc.Should(queryContDesc
.Terms(t => t
.Terms(batchTagIds)
.Field(MapDocumentObjectName(tagField))));
}
queryContainers.Add(queryContDesc.Bool(q => shouldDesc));
}
if (ids != null && ids.Any())
{
queryContainers.Add(queryContDesc.Ids(i => i.Values(ids)));
}
if (dateStart.HasValue)
{
queryContainers.Add(
queryContDesc.DateRange(d => d
.Field(DocumentElastic.ModifiedDateField)
.GreaterThan(dateStart.Value)));
}
if (dateEnd.HasValue)
{
queryContainers.Add(
queryContDesc.DateRange(d => d
.Field(DocumentElastic.ModifiedDateField)
.LessThanOrEquals(dateEnd.Value)));
}
// a REAL _should_ query, if we just add to the queryContainer then at least one of this condition must satisfied
if (!string.IsNullOrEmpty(shouldQuery))
{
var modifiedQuery = shouldQuery;
//replace the field names (because of the document_object)
if (documentObjectFieldNames?.Any() == true)
{
modifiedQuery = PrefixQueryFields(modifiedQuery, documentObjectFieldNames);
}
sdesc.Query(q => q.Bool(b => b
.Must(queryContainers.ToArray())
.Should(sq => sq.QueryString(qs => qs.Query(modifiedQuery)))));
}
else
{
sdesc.Query(q => q.Bool(b => b.Must(queryContainers.ToArray())));
}
if (!string.IsNullOrEmpty(orderBy))
{
var fieldName = MapDocumentObjectName(orderBy);
if (interPretedFields != null && interPretedFields.Contains(orderBy))
{
fieldName += ".raw";
}
sdesc.Sort(s => isDescending ? s.Descending(fieldName) : s.Ascending(fieldName));
}
if (limit >= 0)
{
sdesc.Size(limit);
}
ApplyDocumentFieldFilter(sdesc, returningDocumentObjectFields);
return(GetScrolled(sdesc));
}
public ISearchResponse <DocumentElastic> Search(
AutoCompleteSettingsElastic autoCompleteSettings,
SearchSettingsElastic searchSettings,
string text,
IEnumerable <string> documentObjectFieldNames,
string tagField,
IEnumerable <string> interPretedFields,
FilterElastic defaultFilter,
List <WeightElastic> defaultWeights
)
{
var sdesc = new SearchDescriptor <DocumentElastic>();
#region SEARCH
if (searchSettings?.Count > 0)
{
var queryContDesc = new QueryContainerDescriptor <DocumentElastic>();
var queryContainers = new List <QueryContainer>();
var searchFields = searchSettings.SearchFieldList.Select(f => MapDocumentObjectName(f)).ToArray();
//FILTER
if (searchSettings.UseDefaultFilter)
{
if (!string.IsNullOrEmpty(defaultFilter?.Query))
{
var modifiedQuery = documentObjectFieldNames?.Any() == true?
PrefixQueryFields(defaultFilter.Query, documentObjectFieldNames) :
defaultFilter.Query;
queryContainers.Add(queryContDesc.QueryString(q => q.Query(modifiedQuery)));
}
if (defaultFilter?.TagIdList?.Any() == true)
{
var shouldDesc = new BoolQueryDescriptor <DocumentElastic>();
foreach (var batchTagIds in defaultFilter.TagIdList.Batch(1000))
{
shouldDesc.Should(queryContDesc
.Terms(t => t
.Terms(batchTagIds)
.Field(MapDocumentObjectName(tagField))));
}
queryContainers.Add(queryContDesc.Bool(q => shouldDesc));
}
}
if (!string.IsNullOrEmpty(searchSettings.Filter?.Query))
{
var modifiedQuery = documentObjectFieldNames?.Any() == true?
PrefixQueryFields(searchSettings.Filter.Query, documentObjectFieldNames) :
searchSettings.Filter.Query;
queryContainers.Add(queryContDesc.QueryString(q => q.Query(modifiedQuery)));
}
if (searchSettings.Filter?.TagIdList?.Any() == true)
{
var shouldDesc = new BoolQueryDescriptor <DocumentElastic>();
foreach (var batchTagIds in searchSettings.Filter.TagIdList.Batch(1000))
{
shouldDesc.Should(queryContDesc
.Terms(t => t
.Terms(batchTagIds)
.Field(MapDocumentObjectName(tagField))));
}
queryContainers.Add(queryContDesc.Bool(q => shouldDesc));
}
// MATCH TYPE SEARCH
if (searchSettings.Type == (int)SDK.Net.Models.Enums.SearchTypeEnum.Match)
{
var mqd = new MultiMatchQueryDescriptor <DocumentElastic>()
.Query(text)
.Type(TextQueryType.BestFields)
.CutoffFrequency(searchSettings.CutOffFrequency)
.Fuzziness(searchSettings.Fuzziness < 0 ? Fuzziness.Auto : Fuzziness.EditDistance(searchSettings.Fuzziness))
.Fields(f => f.Fields(searchFields))
.Operator((Operator)searchSettings.Operator);
queryContainers.Add(queryContDesc.MultiMatch(q => mqd));
}
// QUERY(STRING) TYPE SEARCH
if (searchSettings.Type == (int)SDK.Net.Models.Enums.SearchTypeEnum.Query)
{
var modifiedQuery = documentObjectFieldNames?.Any() == true?
PrefixQueryFields(text, documentObjectFieldNames) :
text;
var qsd = new QueryStringQueryDescriptor <DocumentElastic>()
.Query(text)
//cutoff_frequency is not supported for querystring query
//.CutoffFrequency(searchSettings.CutOffFrequency)
.Fuzziness(searchSettings.Fuzziness < 0 ? Fuzziness.Auto : Fuzziness.EditDistance(searchSettings.Fuzziness))
.Fields(f => f.Fields(searchFields))
.DefaultOperator((Operator)searchSettings.Operator);
queryContainers.Add(queryContDesc.QueryString(q => qsd));
}
// WEIGHTS
// a REAL _should_ query, if we just add to the queryContainer then at least one of this condition must satisfied
var weights = new List <WeightElastic>();
if (searchSettings.UseDefaultWeights && (defaultWeights?.Any() == true))
{
weights.AddRange(defaultWeights);
}
if (searchSettings.Weights?.Any() == true)
{
weights.AddRange(searchSettings.Weights);
}
if (weights.Any())
{
var shouldQuery = string.Join(" ", weights.Select(k => $"({k.Query})^{k.Value}"));
var modifiedQuery = documentObjectFieldNames?.Any() == true?
PrefixQueryFields(shouldQuery, documentObjectFieldNames) :
shouldQuery;
sdesc.Query(q => q.Bool(b => b
.Must(queryContainers.ToArray())
.Should(sq => sq.QueryString(qs => qs.Query(modifiedQuery)))));
}
else
{
sdesc.Query(q => q.Bool(b => b.Must(queryContainers.ToArray())));
}
// ORDER
if (!string.IsNullOrEmpty(searchSettings.Order?.OrderByField))
{
var fieldName = MapDocumentObjectName(searchSettings.Order.OrderByField);
if (interPretedFields != null && interPretedFields.Contains(searchSettings.Order.OrderByField))
{
fieldName += ".raw";
}
sdesc.Sort(s => (int)SortOrder.Descending == searchSettings.Order.OrderDirection ? s.Descending(fieldName) : s.Ascending(fieldName));
}
// COUNT
sdesc.Size(searchSettings.Count);
ApplyDocumentFieldFilter(sdesc, searchSettings.ResponseFieldList.Select(f => MapDocumentObjectName(f)));
}
else
{
sdesc.Size(0);
}
#endregion
#region SUGGEST
if (autoCompleteSettings?.Count > 0)
{
var psgd = new PhraseSuggesterDescriptor <DocumentElastic>()
.Field(DocumentElastic.TextField)
.Size(autoCompleteSettings.Count)
//.RealWordErrorLikelihood(0.95), 0.95 is the default value
.Confidence(autoCompleteSettings.Confidence)
.MaxErrors(autoCompleteSettings.MaximumErrors)
.DirectGenerator(dg => dg
.Field(DocumentElastic.TextField)
.SuggestMode(SuggestMode.Always)
.MinWordLength(3)
.MinDocFrequency(3)
.Size(1))
.Collate(c => c
.Prune()
.Query(q => q
//unfortunately Params is not working here so had to hack the text field like this
.Inline($"{{\"match\": {{\"{DocumentElastic.TextField}\" : {{\"query\": \"{{{{suggestion}}}}\", \"operator\": \"and\"}}}}}}")
)
)
.Text(text);
sdesc.Suggest(s => s.Phrase(SuggestName, p => psgd));
}
#endregion
var resp = Client.Search <DocumentElastic>(sdesc);
ResponseValidator(resp);
return(resp);
}
public override async Task <IEnumerable <ArtifactSearchResult> > GetSemanticSearchSuggestions(SearchEngineParameters searchEngineParameters)
{
try
{
var index = await SemanticSearchRepository.GetSemanticSearchIndex();
if (String.IsNullOrEmpty(index))
{
// Returning empty results when index has not been created yet.
return(new List <ArtifactSearchResult>());
}
PerformIndexHealthCheck(index);
// Setting default index name on the connection, otherwise the search request will fail
_elasticClient.ConnectionSettings.DefaultIndices.Clear();
_elasticClient.ConnectionSettings.DefaultIndices.Add(typeof(SemanticSearchItem), index);
var searchText = await SemanticSearchRepository.GetSemanticSearchText(searchEngineParameters.ArtifactId,
searchEngineParameters.UserId);
// Create the bool query descripter that just searchs for the searchText we constructed
var boolQueryDescriptor = new BoolQueryDescriptor <SemanticSearchItem>();
boolQueryDescriptor.Must(GetMoreLikeThisQuery(searchText));
// Dont return back result for the current artifact id we're searching against
boolQueryDescriptor.MustNot(GetArtifactIdMatchQuery(searchEngineParameters.ArtifactId));
// If not instance admin, use the list of accessible project ids to filter out, otherwise no need to filter
if (!searchEngineParameters.IsInstanceAdmin)
{
boolQueryDescriptor.Filter(GetContainsProjectIdsQuery(searchEngineParameters.AccessibleProjectIds));
}
// Creates the search descriptor
var searchDescriptor = new SearchDescriptor <SemanticSearchItem>();
searchDescriptor.Index(index).Size(searchEngineParameters.PageSize).Query(q => q.Bool(b => boolQueryDescriptor));
var results = await _elasticClient.SearchAsync <SemanticSearchItem>(searchDescriptor);
var hits = results.Hits;
var itemIds = new List <int>();
hits.ForEach(a =>
{
int output;
if (Int32.TryParse(a.Id, out output))
{
itemIds.Add(output);
}
});
// parse the artifact ids into a artifactsearchresult to return to the caller
return(await GetArtifactSearchResultsFromItemIds(itemIds.Distinct().ToList(), searchEngineParameters.UserId));
}
catch (Exception ex)
{
if (ex is ElasticsearchConfigurationException)
{
throw;
}
throw new ElasticsearchException(I18NHelper.FormatInvariant("Elastic search failed to process search. Exception:{0}", ex));
}
}
