public static async Task SearchDataInElastic(string type, string term, IHost host)
{
var serviceProvider = host.Services;
var searchClient = serviceProvider.GetRequiredService <ISearchService>();
var loggerFactory = serviceProvider.GetRequiredService <ILoggerFactory>();
var logger = loggerFactory.CreateLogger(typeof(Program));
logger.LogInformation($"Searching {type} for {term}");
var metaData = new SearchMetaData {
Actor = Environment.MachineName, Client = QueryClient.Console
};
switch (type)
{
case "entries":
var entryResult = await searchClient.SearchForEntry(term, 0, 100, metaData);
OutputRenderer.RenderEntriesTable(entryResult);
break;
case "leagues":
var leagueResult = await searchClient.SearchForLeague(term, 0, 100, metaData, "no");
OutputRenderer.RenderLeaguesTable(leagueResult);
break;
}
}
public void ProcessResults_Populates_Search_MetaData()
{
var searchProc = new SearchRequestProcessor <Search> {
BaseUrl = "https://api.twitter.com/1.1/search/"
};
List <Search> results = searchProc.ProcessResults(SearchJson);
Assert.IsNotNull(results);
Assert.IsNotNull(results.SingleOrDefault());
Search search = results.Single();
Assert.IsNotNull(search);
SearchMetaData metaData = search.SearchMetaData;
Assert.IsNotNull(metaData);
Assert.AreEqual(0.046m, metaData.CompletedIn);
Assert.AreEqual(281604225014824960ul, metaData.MaxID);
Assert.AreEqual(5ul, metaData.SinceID);
Assert.AreEqual(10, metaData.Count);
Assert.AreEqual("Twitter", metaData.Query);
Assert.AreEqual(
"?max_id=281562053162979328&q=Twitter&geocode=37.781157%2C-122.398720%2C1mi&include_entities=1",
metaData.NextResults);
Assert.AreEqual(
"?since_id=281604225014824960&q=Twitter&geocode=37.781157%2C-122.398720%2C1mi&include_entities=1",
metaData.RefreshUrl);
}
public static SearchMetaData GetMetaData <T>() where T : class
{
return(metaDataCache.GetOrAdd(
typeof(T),
(type) =>
{
var result = new SearchMetaData();
var properties = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance);
foreach (var property in properties)
{
if (property.CanRead && !property.IsPropertyACollection())
{
result.Fields.Add(
property.Name.ToLower(),
new SearchFieldInfo
{
PropertyInfo = property,
Id = property.HasAttribute <IdAttribute>(),
Keyword = property.HasAttribute <KeywordAttribute>(),
Store = property.HasAttribute <StoreAttribute>(),
Facet = property.HasAttribute <TermFacetAttribute>()
}
);
}
}
return result;
}
));
}
private static SearchMetaData GetSearchMetaData(SlackTeam slackTeam, AppMentionEvent message)
{
var metaData = new SearchMetaData
{
Team = slackTeam?.TeamId, FollowingFplLeagueId = slackTeam?.FplbotLeagueId.ToString(), Actor = message.User,
Client = QueryClient.Slack
};
return(metaData);
}
public async Task <IActionResult> GetAny(string query, int page, SearchType type = SearchType.All)
{
var metaData = new SearchMetaData
{
Client = QueryClient.Web, Actor = Request?.HttpContext.Connection.RemoteIpAddress?.ToString()
};
var searchResult = await _searchService.SearchAny(query, page, 10, metaData, type);
if (searchResult.TotalPages < page && !searchResult.Any())
{
ModelState.AddModelError(nameof(page), $"{nameof(page)} exceeds the total page count");
return(BadRequest(ModelState));
}
return(Ok(new
{
Hits = searchResult,
}));
}
public JsonResult GetSearchMetaData()
{
List <MatrimonyWebsite> allWebSites = new List <MatrimonyWebsite>();
var dbResult = adaptor.GetAllWebSites(out allWebSites);
SearchMetaData searchMetaData = new SearchMetaData
{
Community = allWebSites.Where(webSite => !string.IsNullOrWhiteSpace(webSite.Community)).ToList().GroupBy(webSite => webSite.Community).Select(webSite => webSite.First()).Select(webSite => webSite.Community.ToLower()).ToList(),
MotherTounge = allWebSites.Where(webSite => !string.IsNullOrWhiteSpace(webSite.MotherTounge)).ToList().GroupBy(webSite => webSite.MotherTounge).Select(webSite => webSite.First()).Select(webSite => webSite.MotherTounge.ToLower()).ToList(),
Religion = allWebSites.Where(webSite => !string.IsNullOrWhiteSpace(webSite.Religion)).ToList().GroupBy(webSite => webSite.Religion).Select(webSite => webSite.First()).Select(webSite => webSite.Religion.ToLower()).ToList(),
SpecialCategory = allWebSites.Where(webSite => !string.IsNullOrWhiteSpace(webSite.SpecialCategory)).ToList().GroupBy(webSite => webSite.SpecialCategory).Select(webSite => webSite.First()).Select(webSite => webSite.SpecialCategory.ToLower()).ToList()
};
searchMetaData.Community = searchMetaData.Community.MoveToTop().ToList();
searchMetaData.MotherTounge = searchMetaData.MotherTounge.MoveToTop().ToList();
searchMetaData.Religion = searchMetaData.Religion.MoveToTop().ToList();
searchMetaData.SpecialCategory = searchMetaData.SpecialCategory.MoveToTop().ToList();
return(Json(searchMetaData));
}
public async Task <SearchResult <LeagueItem> > SearchForLeague(string query, int page, int maxHits, SearchMetaData metaData, string countryToBoost = null)
{
var response = await _elasticClient.SearchAsync <LeagueItem>(x => x
.Index(_options.LeaguesIndex)
.From(page *maxHits)
.Size(maxHits)
.Query(q => q
.MultiMatch(m => m
.Fields(f => f.Field(y => y.Name, 1.5).Field(y => y.AdminName))
.Query(query)
.Fuzziness(Fuzziness.Auto))
)
.Sort(GetLeagueSortDescriptor(countryToBoost))
.Preference(metaData?.Actor)
);
_logger.LogInformation("League search for {query} returned {returned} of {hits} hits.", query, response.Hits.Count, response.Total);
await _messageSession.SendLocal(new IndexQuery(DateTime.UtcNow, query, page, _options.LeaguesIndex, countryToBoost, response.Total, response.Took, metaData?.Client.ToString(), metaData?.Team, metaData?.FollowingFplLeagueId, metaData?.Actor));
return(new SearchResult <LeagueItem>(response.Hits.Select(h => h.Source).ToArray(), response.Total, page, maxHits));
}
public async Task <SearchResult <EntryItem> > SearchForEntry(string query, int page, int maxHits, SearchMetaData metaData)
{
var response = await _elasticClient.SearchAsync <EntryItem>(x => x
.Index(_options.EntriesIndex)
.From(page * maxHits)
.Size(maxHits)
.Query(q =>
q.Boosting(a => a
.Positive(p =>
p.MultiMatch(m => m
.Fields(f =>
f.Field(y => y.RealName, 1.5).Field(y => y.TeamName).Field(y => y.Alias).Field(y => y.Description, 0.5))
.Query(query)
.Fuzziness(Fuzziness.Auto)
))
.NegativeBoost(0.3)
.Negative(p => !p.Exists(e => e.Field("verifiedType")))))
.Sort(sd => sd
.Descending(SortSpecialField.Score)
.Ascending(SortSpecialField.DocumentIndexOrder))
.Preference(metaData?.Actor)
);
_logger.LogInformation("Entry search for {query} returned {returned} of {hits} hits.", query, response.Hits.Count, response.Total);
await _messageSession.SendLocal(new IndexQuery(DateTime.UtcNow, query, page, _options.EntriesIndex, null, response.Total, response.Took, metaData?.Client.ToString(), metaData?.Team, metaData?.FollowingFplLeagueId, metaData?.Actor));
return(new SearchResult <EntryItem>(response.Hits.Select(h => h.Source).ToArray(), response.Total, page, maxHits));
}
public async Task <SearchResult <dynamic> > SearchAny(string query, int page, int maxHits, SearchMetaData metaData)
{
var response = await _elasticClient.SearchAsync <ILazyDocument>(s => s
.Index($"{_options.EntriesIndex},{_options.LeaguesIndex}")
.From(page *maxHits)
.Size(maxHits)
.Query(q =>
{
// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-boosting-query.html#boosting-top-level-params
q.Boosting(a => a
.Positive(p =>
{
return(p.MultiMatch(dog => dog
.Fields(f => f
.Field("realName", 10) // entry
.Field("name", 8) // league
.Field("teamName", 3) // league
.Field("adminName", 2) // league
.Field("adminTeamName")) // league
.Query(string.IsNullOrEmpty(query) ? "*" : query)
.Fuzziness(Fuzziness.Auto)));
})
.NegativeBoost(0.3).Negative(p => !p.Exists(e => e.Field("verifiedType"))));
return(q);
})
.Sort(sd => sd
.Descending(SortSpecialField.Score)
.Ascending(SortSpecialField.DocumentIndexOrder)
)
.Preference(metaData?.Actor));
if (!response.IsValid)
{
throw new Exception(response.DebugInformation, response.OriginalException);
}
return(new SearchResult <dynamic>(response.Hits.Select(h =>
{
string type = h.Index.Split("-")[1] == "entries" ? "entry" : "league";
if (h.Index == _options.EntriesIndex)
{
return new SearchContainer
{
Type = type,
Source = h.Source.As <EntryItem>()
};
}
if (h.Index == _options.LeaguesIndex)
{
return new SearchContainer
{
Type = type,
Source = h.Source.As <LeagueItem>()
}
}
;
return new SearchContainer {
Source = h.Source
};
}).ToArray(), response.Total, page, maxHits));
}
public async Task <SearchResult <dynamic> > SearchAny(string query, int page, int maxHits, SearchMetaData metaData, SearchType searchType = SearchType.All)
{
var indexPattern = GetIndexPatternToSearch(searchType);
var response = await _elasticClient.SearchAsync <ILazyDocument>(s => s
.Index(indexPattern)
.From(page *maxHits)
.Size(maxHits)
.Query(q =>
{
// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-boosting-query.html#boosting-top-level-params
q.Boosting(a => a
.Positive(p =>
{
return(p.QueryString(dog => dog
.Fields(f => f
.Field("realName", 15) // entry
.Field("name", 1) // league
.Field("alias", 5) // league
.Field("teamName", 3) // league
.Field("adminName", 5) // league
.Field("adminTeamName")) // league
.Query(string.IsNullOrEmpty(query) ? "*" : query)
.Escape()
.Fuzziness(Fuzziness.Auto)));
})
.Negative(p => !p.Exists(e => e.Field("verifiedType")))
.NegativeBoost(0.1));
return(q);
})
.Sort(sd => sd
.Descending(SortSpecialField.Score)
.Ascending(SortSpecialField.DocumentIndexOrder)
)
.Preference(metaData?.Actor));
if (!response.IsValid)
{
throw new Exception(response.DebugInformation, response.OriginalException);
}
await _messageSession.SendLocal(new IndexQuery(DateTime.UtcNow, query, page, indexPattern, null, response.Total, response.Took, metaData?.Client.ToString(), metaData?.Team, metaData?.FollowingFplLeagueId, metaData?.Actor));
return(new SearchResult <dynamic>(response.Hits.Select(h =>
{
if (h.Index == _options.EntriesIndex)
{
return new SearchContainer
{
Type = "entry",
Source = h.Source.As <EntryItem>()
};
}
if (h.Index == _options.LeaguesIndex)
{
return new SearchContainer
{
Type = "league",
Source = h.Source.As <LeagueItem>()
}
}
;
return new SearchContainer {
Source = h.Source
};
}).ToArray(), response.Total, page, maxHits));
}
public LuceneSearchIndex()
{
indexType = typeof(T).AssemblyQualifiedName.SafeFilename();
searchMetaData = SearchMetaDataUtility.GetMetaData <T>();
}
