public static ElasticSearchOptions <T> WithPaging <T>(this ElasticSearchOptions <T> options, PagingOptions paging) where T : class
{
if (paging == null)
{
return(options);
}
var elasticSearchPagingOptions = paging as ElasticSearchPagingOptions <T>;
if (elasticSearchPagingOptions != null)
{
options.BeforeQuery = elasticSearchPagingOptions.BeforeFilter;
options.AfterQuery = elasticSearchPagingOptions.AfterFilter;
options.SortBy.AddRange(elasticSearchPagingOptions.SortBy);
}
options.BeforeValue = paging.Before;
options.AfterValue = paging.After;
options.Page = paging.Page;
options.Limit = paging.Limit;
options.HasMore = false;
options.HasMoreChanged += (source, args) => paging.HasMore = args.Value;
return(options);
}
public static FilterContainer GetElasticSearchFilter <T>(this ElasticSearchOptions <T> options) where T : class
{
var queries = GetElasticSearchFilter <T>((QueryOptions)options);
if (options.UseDateRange)
{
queries &= Filter <T> .Range(r => r.OnField(options.DateField).GreaterOrEquals(options.GetStartDate()).LowerOrEquals(options.GetEndDate()));
}
if (!String.IsNullOrEmpty(options.BeforeValue) && options.BeforeQuery == null)
{
options.BeforeQuery = Filter <T> .Range(r => r.OnField("_uid").Lower(options.BeforeValue));
}
if (!String.IsNullOrEmpty(options.AfterValue) && options.AfterQuery == null)
{
options.AfterQuery = Filter <T> .Range(r => r.OnField("_uid").Greater(options.AfterValue));
}
if (options.BeforeQuery != null)
{
queries &= options.BeforeQuery;
}
if (options.AfterQuery != null)
{
queries &= options.AfterQuery;
}
return(queries);
}
public ICollection <Stack> GetMostRecent(string projectId, DateTime utcStart, DateTime utcEnd, PagingOptions paging, bool includeHidden = false, bool includeFixed = false, bool includeNotFound = true)
{
var options = new ElasticSearchOptions <Stack>().WithProjectId(projectId).WithSort(s => s.OnField(e => e.LastOccurrence).Descending()).WithPaging(paging);
options.Filter = Filter <Stack> .Range(r => r.OnField(s => s.LastOccurrence).GreaterOrEquals(utcStart));
options.Filter &= Filter <Stack> .Range(r => r.OnField(s => s.LastOccurrence).LowerOrEquals(utcEnd));
if (!includeFixed)
{
options.Filter &= Filter <Stack> .Missing(s => s.DateFixed);
}
if (!includeHidden)
{
options.Filter &= !Filter <Stack> .Term(s => s.IsHidden, true);
}
if (!includeNotFound)
{
options.Filter &= Filter <Stack> .Missing("signature_info.path");
}
return(Find(options));
}
public Task <FindResults <T> > GetAllAsync(string sort = null, SortOrder sortOrder = SortOrder.Ascending, PagingOptions paging = null)
{
var search = new ElasticSearchOptions <T>()
.WithPaging(paging)
.WithSort(sort, sortOrder);
return(FindAsync(search));
}
public static ElasticSearchOptions <T> WithSort <T>(this ElasticSearchOptions <T> options, string sort, SortOrder sortOrder) where T : class
{
if (!String.IsNullOrEmpty(sort))
{
options.WithSort(e => e.OnField(sort).Order(sortOrder == SortOrder.Descending ? Nest.SortOrder.Descending : Nest.SortOrder.Ascending));
}
return(options);
}
public static FilterContainer GetElasticSearchFilter <T>(this ElasticSearchOptions <T> options, bool supportSoftDeletes = false) where T : class
{
var container = Filter <T> .MatchAll();
container = ApplyQueryOptionsFilters <T>(options, container, supportSoftDeletes);
container = ApplyElasticSearchOptionsFilters(options, container);
return(container);
}
public Task <long> CountByOrganizationIdAsync(string organizationId)
{
var filter = Filter <User> .Term(u => u.OrganizationIds, new[] { organizationId });
var options = new ElasticSearchOptions <User>()
.WithFilter(filter);
return(CountAsync(options));
}
public static QueryContainer GetElasticSearchQuery <T>(this ElasticSearchOptions <T> options, bool supportSoftDeletes = false) where T : class
{
var filterContainer = Filter <T> .MatchAll();
filterContainer = ApplyQueryOptionsFilters <T>(options, filterContainer, supportSoftDeletes);
filterContainer = ApplyElasticSearchOptionsFilters(options, filterContainer);
return(Query <T> .Filtered(f => f.Filter(d => filterContainer)));
}
public static ElasticSearchOptions <T> WithIndicesFromDateRange <T>(this ElasticSearchOptions <T> options, string nameFormat = null) where T : class
{
if (!options.UseDateRange)
{
return(options);
}
options.Indices.AddRange(GetTargetIndex <T>(options.GetStartDate(), options.GetEndDate(), nameFormat));
return(options);
}
public static ElasticSearchOptions <T> WithIndicesFromDateRange <T>(this ElasticSearchOptions <T> options) where T : PersistentEvent
{
if (!options.UseDateRange)
{
return(options);
}
options.Indices.AddRange(GetTargetIndex(options.GetStartDate(), options.GetEndDate()));
return(options);
}
public Task <FindResults <Stack> > GetMostRecentAsync(string projectId, DateTime utcStart, DateTime utcEnd, PagingOptions paging, string query)
{
var options = new ElasticSearchOptions <Stack>().WithProjectId(projectId).WithQuery(query).WithSort(s => s.OnField(e => e.LastOccurrence).Descending()).WithPaging(paging);
options.Filter = Filter <Stack> .Range(r => r.OnField(s => s.LastOccurrence).GreaterOrEquals(utcStart));
options.Filter &= Filter <Stack> .Range(r => r.OnField(s => s.LastOccurrence).LowerOrEquals(utcEnd));
return(FindAsync(options));
}
public ICollection <Stack> GetNew(string projectId, DateTime utcStart, DateTime utcEnd, PagingOptions paging, string query)
{
var options = new ElasticSearchOptions <Stack>().WithProjectId(projectId).WithQuery(query).WithSort(s => s.OnField(e => e.FirstOccurrence).Descending()).WithPaging(paging);
options.Filter = Filter <Stack> .Range(r => r.OnField(s => s.FirstOccurrence).GreaterOrEquals(utcStart));
options.Filter &= Filter <Stack> .Range(r => r.OnField(s => s.FirstOccurrence).LowerOrEquals(utcEnd));
return(Find(options));
}
public Task <FindResults <T> > GetBySearchAsync(string systemFilter, string userFilter = null, string query = null, string sort = null, SortOrder sortOrder = SortOrder.Ascending, PagingOptions paging = null)
{
var search = new ElasticSearchOptions <T>()
.WithSystemFilter(systemFilter)
.WithFilter(userFilter)
.WithQuery(query, false)
.WithSort(sort, sortOrder)
.WithPaging(paging);
return(FindAsync(search));
}
private static FilterContainer ApplyElasticSearchOptionsFilters <T>(ElasticSearchOptions <T> options, FilterContainer container, bool isQuery = false) where T : class
{
if (container == null)
{
container = Filter <T> .MatchAll();
}
if (options?.SystemFilter != null)
{
container &= options.SystemFilter;
}
if (options?.Filter != null)
{
container &= options.Filter;
}
if (options != null && options.UseDateRange)
{
container &= Filter <T> .Range(r => r.OnField(options.DateField).GreaterOrEquals(options.GetStartDate()).LowerOrEquals(options.GetEndDate()));
}
if (!String.IsNullOrEmpty(options?.BeforeValue) && options.BeforeQuery == null)
{
options.BeforeQuery = Filter <T> .Range(r => r.OnField("_uid").Lower(options.BeforeValue));
}
if (!String.IsNullOrEmpty(options?.AfterValue) && options.AfterQuery == null)
{
options.AfterQuery = Filter <T> .Range(r => r.OnField("_uid").Greater(options.AfterValue));
}
if (options?.BeforeQuery != null)
{
container &= options.BeforeQuery;
}
if (options?.AfterQuery != null)
{
container &= options.AfterQuery;
}
if (!isQuery && !String.IsNullOrEmpty(options?.Query))
{
container &= Filter <T> .Query(q => q.QueryString(qs => qs.DefaultOperator(options.DefaultQueryOperator).Query(options.Query).AnalyzeWildcard()));
}
return(container);
}
protected async Task <long> CountAsync(ElasticSearchOptions <T> options)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (EnableCache && options.UseCache)
{
var cachedValue = await Cache.GetAsync <long>(GetScopedCacheKey("count-" + options.CacheKey)).AnyContext();
if (cachedValue.HasValue)
{
return(cachedValue.Value);
}
}
var countDescriptor = new CountDescriptor <T>().Query(f => f.Filtered(s => s.Filter(f2 => options.GetElasticSearchFilter(_supportsSoftDeletes))));
countDescriptor.Indices(options.Indices.Any()
? options.Indices.ToArray()
: GetIndices());
countDescriptor.IgnoreUnavailable();
countDescriptor.Type(typeof(T));
#if DEBUG
_elasticClient.EnableTrace();
var sw = Stopwatch.StartNew();
#endif
var results = await _elasticClient.CountAsync <T>(countDescriptor).AnyContext();
#if DEBUG
sw.Stop();
_elasticClient.DisableTrace();
Logger.Trace().Message($"CountAsync: {sw.ElapsedMilliseconds}ms, Serialization Took {results.ConnectionStatus.Metrics.SerializationTime}ms, Deserialization Took {results.ConnectionStatus.Metrics.DeserializationTime}ms").Write();
#endif
if (!results.IsValid)
{
throw new ApplicationException($"ElasticSearch error code \"{results.ConnectionStatus.HttpStatusCode}\".", results.ConnectionStatus.OriginalException);
}
if (EnableCache && options.UseCache)
{
await Cache.SetAsync(GetScopedCacheKey("count-" + options.CacheKey), results.Count, options.GetCacheExpirationDate()).AnyContext();
}
return(results.Count);
}
public ICollection <Stack> GetByFilter(string systemFilter, string userFilter, string sort, SortOrder sortOrder, string field, DateTime utcStart, DateTime utcEnd, PagingOptions paging)
{
if (String.IsNullOrEmpty(sort))
{
sort = "last";
sortOrder = SortOrder.Descending;
}
var search = new ElasticSearchOptions <Stack>()
.WithDateRange(utcStart, utcEnd, field ?? "last")
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <Stack> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithPaging(paging)
.WithSort(e => e.OnField(sort).Order(sortOrder == SortOrder.Descending ? Nest.SortOrder.Descending : Nest.SortOrder.Ascending));
return(Find(search));
}
public Task <FindResults <PersistentEvent> > GetByFilterAsync(string systemFilter, string userFilter, string sort, SortOrder sortOrder, string field, DateTime utcStart, DateTime utcEnd, PagingOptions paging)
{
if (String.IsNullOrEmpty(sort))
{
sort = "date";
sortOrder = SortOrder.Descending;
}
var search = new ElasticSearchOptions <PersistentEvent>()
.WithDateRange(utcStart, utcEnd, field ?? "date")
.WithIndicesFromDateRange($"'{_index.VersionedName}-'yyyyMM")
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithPaging(paging)
.WithSort(e => e.OnField(sort).Order(sortOrder == SortOrder.Descending ? Nest.SortOrder.Descending : Nest.SortOrder.Ascending));
return(FindAsync(search));
}
protected long Count(ElasticSearchOptions <T> options)
{
if (options == null)
{
throw new ArgumentNullException("options");
}
long?result;
if (EnableCache && options.UseCache)
{
result = Cache.Get <long?>(GetScopedCacheKey("count-" + options.CacheKey));
if (result.HasValue)
{
return(result.Value);
}
}
var countDescriptor = new CountDescriptor <T>().Query(f => f.Filtered(s => s.Filter(f2 => options.GetElasticSearchFilter())));
countDescriptor.Indices(options.Indices);
countDescriptor.IgnoreUnavailable();
countDescriptor.Type(typeof(T));
_elasticClient.EnableTrace();
var results = _elasticClient.Count <T>(countDescriptor);
_elasticClient.DisableTrace();
if (!results.IsValid)
{
throw new ApplicationException(String.Format("ElasticSearch error code \"{0}\".", results.ConnectionStatus.HttpStatusCode), results.ConnectionStatus.OriginalException);
}
result = results.Count;
if (EnableCache && options.UseCache)
{
Cache.Set(GetScopedCacheKey("count-" + options.CacheKey), result, options.GetCacheExpirationDate());
}
return(result.Value);
}
protected async Task <IDictionary <string, long> > SimpleAggregationAsync(ElasticSearchOptions <T> options, Expression <Func <T, object> > fieldExpression)
{
var searchDescriptor = new SearchDescriptor <T>()
.Query(f => f.Filtered(s => s.Filter(f2 => options.GetElasticSearchFilter(_supportsSoftDeletes))))
.Aggregations(a => a.Terms("simple", sel => sel.Field(fieldExpression).Size(10)));
searchDescriptor.Indices(options.Indices.Any()
? options.Indices.ToArray()
: GetIndices());
#if DEBUG
_elasticClient.EnableTrace();
var sw = Stopwatch.StartNew();
#endif
var results = await _elasticClient.SearchAsync <T>(searchDescriptor).AnyContext();
#if DEBUG
sw.Stop();
_elasticClient.DisableTrace();
Logger.Trace().Message($"SimpleAggregationAsync: {sw.ElapsedMilliseconds}ms, Elastic Took {results.ElapsedMilliseconds}ms, Serialization Took {results.ConnectionStatus.Metrics.SerializationTime}ms, Deserialization Took {results.ConnectionStatus.Metrics.DeserializationTime}ms").Write();
#endif
return(results.Aggs.Terms("simple").Items.ToDictionary(ar => ar.Key, ar => ar.DocCount));
}
public static ElasticSearchOptions <T> WithQuery <T>(this ElasticSearchOptions <T> options, string query) where T : class
{
options.Query = query;
return(options);
}
public static ElasticSearchOptions <T> WithFilter <T>(this ElasticSearchOptions <T> options, FilterContainer filter) where T : class
{
options.Filter = filter;
return(options);
}
public static ElasticSearchOptions <T> WithIndices <T>(this ElasticSearchOptions <T> options, DateTime?utcStart, DateTime?utcEnd) where T : class
{
options.Indices.AddRange(GetTargetIndex(utcStart, utcEnd));
return(options);
}
public static ElasticSearchOptions <T> WithSort <T>(this ElasticSearchOptions <T> options, Func <SortFieldDescriptor <T>, IFieldSort> sort) where T : class
{
options.SortBy.Add(sort);
return(options);
}
public static ElasticSearchOptions <T> WithQuery <T>(this ElasticSearchOptions <T> options, string query, bool useAndAsDefaultOperator = true) where T : class
{
options.Query = query;
options.DefaultQueryOperator = useAndAsDefaultOperator ? Operator.And : Operator.Or;
return(options);
}
public static ElasticSearchOptions <T> WithIndices <T>(this ElasticSearchOptions <T> options, DateTime?utcStart, DateTime?utcEnd, string nameFormat = null) where T : class
{
options.Indices.AddRange(GetTargetIndex <T>(utcStart, utcEnd, nameFormat));
return(options);
}
public static ElasticSearchOptions <T> WithIndices <T>(this ElasticSearchOptions <T> options, IEnumerable <string> indices) where T : class
{
options.Indices.AddRange(indices);
return(options);
}
public static ElasticSearchOptions <T> WithIndex <T>(this ElasticSearchOptions <T> options, string index) where T : class
{
options.Indices.Add(index);
return(options);
}
public static ElasticSearchOptions <T> WithSystemFilter <T>(this ElasticSearchOptions <T> options, string filter) where T : class
{
options.SystemFilter = !String.IsNullOrEmpty(filter) ? Filter <T> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(filter))) : null;
return(options);
}
protected ICollection <T> Find(ElasticSearchOptions <T> options)
{
return(FindAs <T>(options));
}
protected ICollection <TModel> FindAs <TModel>(ElasticSearchOptions <T> options) where TModel : class, new()
{
if (options == null)
{
throw new ArgumentNullException("options");
}
ICollection <TModel> result;
if (EnableCache && options.UseCache)
{
result = Cache.Get <ICollection <TModel> >(GetScopedCacheKey(options.CacheKey));
if (result != null)
{
return(result);
}
}
var searchDescriptor = new SearchDescriptor <T>().Filter(options.GetElasticSearchFilter());
searchDescriptor.Indices(options.Indices);
searchDescriptor.IgnoreUnavailable();
if (options.UsePaging)
{
searchDescriptor.Skip(options.GetSkip());
}
searchDescriptor.Size(options.GetLimit());
searchDescriptor.Type(typeof(T));
if (options.Fields.Count > 0)
{
searchDescriptor.Source(s => s.Include(options.Fields.ToArray()));
}
else
{
searchDescriptor.Source(s => s.Exclude("idx"));
}
if (options.SortBy.Count > 0)
{
foreach (var sort in options.SortBy)
{
searchDescriptor.Sort(sort);
}
}
_elasticClient.EnableTrace();
var results = _elasticClient.Search <T>(searchDescriptor);
_elasticClient.DisableTrace();
if (!results.IsValid)
{
throw new ApplicationException(String.Format("ElasticSearch error code \"{0}\".", results.ConnectionStatus.HttpStatusCode), results.ConnectionStatus.OriginalException);
}
options.HasMore = options.UseLimit && results.Total > options.GetLimit();
var items = results.Documents.ToList();
if (typeof(T) != typeof(TModel))
{
if (Mapper.FindTypeMapFor <T, TModel>() == null)
{
Mapper.CreateMap <T, TModel>();
}
result = Enumerable.ToList(items.Select(Mapper.Map <T, TModel>));
}
else
{
result = items as List <TModel>;
}
if (EnableCache && options.UseCache)
{
Cache.Set(GetScopedCacheKey(options.CacheKey), result, options.GetCacheExpirationDate());
}
return(result);
}