public static ElasticSearchOptions <T> WithPaging <T>(this ElasticSearchOptions <T> options, PagingOptions paging) where T : class { if (paging == null) { return(options); } var elasticSearchPagingOptions = paging as ElasticSearchPagingOptions <T>; if (elasticSearchPagingOptions != null) { options.BeforeQuery = elasticSearchPagingOptions.BeforeFilter; options.AfterQuery = elasticSearchPagingOptions.AfterFilter; options.SortBy.AddRange(elasticSearchPagingOptions.SortBy); } options.BeforeValue = paging.Before; options.AfterValue = paging.After; options.Page = paging.Page; options.Limit = paging.Limit; options.HasMore = false; options.HasMoreChanged += (source, args) => paging.HasMore = args.Value; return(options); }
public static FilterContainer GetElasticSearchFilter <T>(this ElasticSearchOptions <T> options) where T : class { var queries = GetElasticSearchFilter <T>((QueryOptions)options); if (options.UseDateRange) { queries &= Filter <T> .Range(r => r.OnField(options.DateField).GreaterOrEquals(options.GetStartDate()).LowerOrEquals(options.GetEndDate())); } if (!String.IsNullOrEmpty(options.BeforeValue) && options.BeforeQuery == null) { options.BeforeQuery = Filter <T> .Range(r => r.OnField("_uid").Lower(options.BeforeValue)); } if (!String.IsNullOrEmpty(options.AfterValue) && options.AfterQuery == null) { options.AfterQuery = Filter <T> .Range(r => r.OnField("_uid").Greater(options.AfterValue)); } if (options.BeforeQuery != null) { queries &= options.BeforeQuery; } if (options.AfterQuery != null) { queries &= options.AfterQuery; } return(queries); }
public ICollection <Stack> GetMostRecent(string projectId, DateTime utcStart, DateTime utcEnd, PagingOptions paging, bool includeHidden = false, bool includeFixed = false, bool includeNotFound = true) { var options = new ElasticSearchOptions <Stack>().WithProjectId(projectId).WithSort(s => s.OnField(e => e.LastOccurrence).Descending()).WithPaging(paging); options.Filter = Filter <Stack> .Range(r => r.OnField(s => s.LastOccurrence).GreaterOrEquals(utcStart)); options.Filter &= Filter <Stack> .Range(r => r.OnField(s => s.LastOccurrence).LowerOrEquals(utcEnd)); if (!includeFixed) { options.Filter &= Filter <Stack> .Missing(s => s.DateFixed); } if (!includeHidden) { options.Filter &= !Filter <Stack> .Term(s => s.IsHidden, true); } if (!includeNotFound) { options.Filter &= Filter <Stack> .Missing("signature_info.path"); } return(Find(options)); }
public Task <FindResults <T> > GetAllAsync(string sort = null, SortOrder sortOrder = SortOrder.Ascending, PagingOptions paging = null) { var search = new ElasticSearchOptions <T>() .WithPaging(paging) .WithSort(sort, sortOrder); return(FindAsync(search)); }
public static ElasticSearchOptions <T> WithSort <T>(this ElasticSearchOptions <T> options, string sort, SortOrder sortOrder) where T : class { if (!String.IsNullOrEmpty(sort)) { options.WithSort(e => e.OnField(sort).Order(sortOrder == SortOrder.Descending ? Nest.SortOrder.Descending : Nest.SortOrder.Ascending)); } return(options); }
public static FilterContainer GetElasticSearchFilter <T>(this ElasticSearchOptions <T> options, bool supportSoftDeletes = false) where T : class { var container = Filter <T> .MatchAll(); container = ApplyQueryOptionsFilters <T>(options, container, supportSoftDeletes); container = ApplyElasticSearchOptionsFilters(options, container); return(container); }
public Task <long> CountByOrganizationIdAsync(string organizationId) { var filter = Filter <User> .Term(u => u.OrganizationIds, new[] { organizationId }); var options = new ElasticSearchOptions <User>() .WithFilter(filter); return(CountAsync(options)); }
public static QueryContainer GetElasticSearchQuery <T>(this ElasticSearchOptions <T> options, bool supportSoftDeletes = false) where T : class { var filterContainer = Filter <T> .MatchAll(); filterContainer = ApplyQueryOptionsFilters <T>(options, filterContainer, supportSoftDeletes); filterContainer = ApplyElasticSearchOptionsFilters(options, filterContainer); return(Query <T> .Filtered(f => f.Filter(d => filterContainer))); }
public static ElasticSearchOptions <T> WithIndicesFromDateRange <T>(this ElasticSearchOptions <T> options, string nameFormat = null) where T : class { if (!options.UseDateRange) { return(options); } options.Indices.AddRange(GetTargetIndex <T>(options.GetStartDate(), options.GetEndDate(), nameFormat)); return(options); }
public static ElasticSearchOptions <T> WithIndicesFromDateRange <T>(this ElasticSearchOptions <T> options) where T : PersistentEvent { if (!options.UseDateRange) { return(options); } options.Indices.AddRange(GetTargetIndex(options.GetStartDate(), options.GetEndDate())); return(options); }
public Task <FindResults <Stack> > GetMostRecentAsync(string projectId, DateTime utcStart, DateTime utcEnd, PagingOptions paging, string query) { var options = new ElasticSearchOptions <Stack>().WithProjectId(projectId).WithQuery(query).WithSort(s => s.OnField(e => e.LastOccurrence).Descending()).WithPaging(paging); options.Filter = Filter <Stack> .Range(r => r.OnField(s => s.LastOccurrence).GreaterOrEquals(utcStart)); options.Filter &= Filter <Stack> .Range(r => r.OnField(s => s.LastOccurrence).LowerOrEquals(utcEnd)); return(FindAsync(options)); }
public ICollection <Stack> GetNew(string projectId, DateTime utcStart, DateTime utcEnd, PagingOptions paging, string query) { var options = new ElasticSearchOptions <Stack>().WithProjectId(projectId).WithQuery(query).WithSort(s => s.OnField(e => e.FirstOccurrence).Descending()).WithPaging(paging); options.Filter = Filter <Stack> .Range(r => r.OnField(s => s.FirstOccurrence).GreaterOrEquals(utcStart)); options.Filter &= Filter <Stack> .Range(r => r.OnField(s => s.FirstOccurrence).LowerOrEquals(utcEnd)); return(Find(options)); }
public Task <FindResults <T> > GetBySearchAsync(string systemFilter, string userFilter = null, string query = null, string sort = null, SortOrder sortOrder = SortOrder.Ascending, PagingOptions paging = null) { var search = new ElasticSearchOptions <T>() .WithSystemFilter(systemFilter) .WithFilter(userFilter) .WithQuery(query, false) .WithSort(sort, sortOrder) .WithPaging(paging); return(FindAsync(search)); }
private static FilterContainer ApplyElasticSearchOptionsFilters <T>(ElasticSearchOptions <T> options, FilterContainer container, bool isQuery = false) where T : class { if (container == null) { container = Filter <T> .MatchAll(); } if (options?.SystemFilter != null) { container &= options.SystemFilter; } if (options?.Filter != null) { container &= options.Filter; } if (options != null && options.UseDateRange) { container &= Filter <T> .Range(r => r.OnField(options.DateField).GreaterOrEquals(options.GetStartDate()).LowerOrEquals(options.GetEndDate())); } if (!String.IsNullOrEmpty(options?.BeforeValue) && options.BeforeQuery == null) { options.BeforeQuery = Filter <T> .Range(r => r.OnField("_uid").Lower(options.BeforeValue)); } if (!String.IsNullOrEmpty(options?.AfterValue) && options.AfterQuery == null) { options.AfterQuery = Filter <T> .Range(r => r.OnField("_uid").Greater(options.AfterValue)); } if (options?.BeforeQuery != null) { container &= options.BeforeQuery; } if (options?.AfterQuery != null) { container &= options.AfterQuery; } if (!isQuery && !String.IsNullOrEmpty(options?.Query)) { container &= Filter <T> .Query(q => q.QueryString(qs => qs.DefaultOperator(options.DefaultQueryOperator).Query(options.Query).AnalyzeWildcard())); } return(container); }
protected async Task <long> CountAsync(ElasticSearchOptions <T> options) { if (options == null) { throw new ArgumentNullException(nameof(options)); } if (EnableCache && options.UseCache) { var cachedValue = await Cache.GetAsync <long>(GetScopedCacheKey("count-" + options.CacheKey)).AnyContext(); if (cachedValue.HasValue) { return(cachedValue.Value); } } var countDescriptor = new CountDescriptor <T>().Query(f => f.Filtered(s => s.Filter(f2 => options.GetElasticSearchFilter(_supportsSoftDeletes)))); countDescriptor.Indices(options.Indices.Any() ? options.Indices.ToArray() : GetIndices()); countDescriptor.IgnoreUnavailable(); countDescriptor.Type(typeof(T)); #if DEBUG _elasticClient.EnableTrace(); var sw = Stopwatch.StartNew(); #endif var results = await _elasticClient.CountAsync <T>(countDescriptor).AnyContext(); #if DEBUG sw.Stop(); _elasticClient.DisableTrace(); Logger.Trace().Message($"CountAsync: {sw.ElapsedMilliseconds}ms, Serialization Took {results.ConnectionStatus.Metrics.SerializationTime}ms, Deserialization Took {results.ConnectionStatus.Metrics.DeserializationTime}ms").Write(); #endif if (!results.IsValid) { throw new ApplicationException($"ElasticSearch error code \"{results.ConnectionStatus.HttpStatusCode}\".", results.ConnectionStatus.OriginalException); } if (EnableCache && options.UseCache) { await Cache.SetAsync(GetScopedCacheKey("count-" + options.CacheKey), results.Count, options.GetCacheExpirationDate()).AnyContext(); } return(results.Count); }
public ICollection <Stack> GetByFilter(string systemFilter, string userFilter, string sort, SortOrder sortOrder, string field, DateTime utcStart, DateTime utcEnd, PagingOptions paging) { if (String.IsNullOrEmpty(sort)) { sort = "last"; sortOrder = SortOrder.Descending; } var search = new ElasticSearchOptions <Stack>() .WithDateRange(utcStart, utcEnd, field ?? "last") .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <Stack> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithPaging(paging) .WithSort(e => e.OnField(sort).Order(sortOrder == SortOrder.Descending ? Nest.SortOrder.Descending : Nest.SortOrder.Ascending)); return(Find(search)); }
public Task <FindResults <PersistentEvent> > GetByFilterAsync(string systemFilter, string userFilter, string sort, SortOrder sortOrder, string field, DateTime utcStart, DateTime utcEnd, PagingOptions paging) { if (String.IsNullOrEmpty(sort)) { sort = "date"; sortOrder = SortOrder.Descending; } var search = new ElasticSearchOptions <PersistentEvent>() .WithDateRange(utcStart, utcEnd, field ?? "date") .WithIndicesFromDateRange($"'{_index.VersionedName}-'yyyyMM") .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithPaging(paging) .WithSort(e => e.OnField(sort).Order(sortOrder == SortOrder.Descending ? Nest.SortOrder.Descending : Nest.SortOrder.Ascending)); return(FindAsync(search)); }
protected long Count(ElasticSearchOptions <T> options) { if (options == null) { throw new ArgumentNullException("options"); } long?result; if (EnableCache && options.UseCache) { result = Cache.Get <long?>(GetScopedCacheKey("count-" + options.CacheKey)); if (result.HasValue) { return(result.Value); } } var countDescriptor = new CountDescriptor <T>().Query(f => f.Filtered(s => s.Filter(f2 => options.GetElasticSearchFilter()))); countDescriptor.Indices(options.Indices); countDescriptor.IgnoreUnavailable(); countDescriptor.Type(typeof(T)); _elasticClient.EnableTrace(); var results = _elasticClient.Count <T>(countDescriptor); _elasticClient.DisableTrace(); if (!results.IsValid) { throw new ApplicationException(String.Format("ElasticSearch error code \"{0}\".", results.ConnectionStatus.HttpStatusCode), results.ConnectionStatus.OriginalException); } result = results.Count; if (EnableCache && options.UseCache) { Cache.Set(GetScopedCacheKey("count-" + options.CacheKey), result, options.GetCacheExpirationDate()); } return(result.Value); }
protected async Task <IDictionary <string, long> > SimpleAggregationAsync(ElasticSearchOptions <T> options, Expression <Func <T, object> > fieldExpression) { var searchDescriptor = new SearchDescriptor <T>() .Query(f => f.Filtered(s => s.Filter(f2 => options.GetElasticSearchFilter(_supportsSoftDeletes)))) .Aggregations(a => a.Terms("simple", sel => sel.Field(fieldExpression).Size(10))); searchDescriptor.Indices(options.Indices.Any() ? options.Indices.ToArray() : GetIndices()); #if DEBUG _elasticClient.EnableTrace(); var sw = Stopwatch.StartNew(); #endif var results = await _elasticClient.SearchAsync <T>(searchDescriptor).AnyContext(); #if DEBUG sw.Stop(); _elasticClient.DisableTrace(); Logger.Trace().Message($"SimpleAggregationAsync: {sw.ElapsedMilliseconds}ms, Elastic Took {results.ElapsedMilliseconds}ms, Serialization Took {results.ConnectionStatus.Metrics.SerializationTime}ms, Deserialization Took {results.ConnectionStatus.Metrics.DeserializationTime}ms").Write(); #endif return(results.Aggs.Terms("simple").Items.ToDictionary(ar => ar.Key, ar => ar.DocCount)); }
public static ElasticSearchOptions <T> WithQuery <T>(this ElasticSearchOptions <T> options, string query) where T : class { options.Query = query; return(options); }
public static ElasticSearchOptions <T> WithFilter <T>(this ElasticSearchOptions <T> options, FilterContainer filter) where T : class { options.Filter = filter; return(options); }
public static ElasticSearchOptions <T> WithIndices <T>(this ElasticSearchOptions <T> options, DateTime?utcStart, DateTime?utcEnd) where T : class { options.Indices.AddRange(GetTargetIndex(utcStart, utcEnd)); return(options); }
public static ElasticSearchOptions <T> WithSort <T>(this ElasticSearchOptions <T> options, Func <SortFieldDescriptor <T>, IFieldSort> sort) where T : class { options.SortBy.Add(sort); return(options); }
public static ElasticSearchOptions <T> WithQuery <T>(this ElasticSearchOptions <T> options, string query, bool useAndAsDefaultOperator = true) where T : class { options.Query = query; options.DefaultQueryOperator = useAndAsDefaultOperator ? Operator.And : Operator.Or; return(options); }
public static ElasticSearchOptions <T> WithIndices <T>(this ElasticSearchOptions <T> options, DateTime?utcStart, DateTime?utcEnd, string nameFormat = null) where T : class { options.Indices.AddRange(GetTargetIndex <T>(utcStart, utcEnd, nameFormat)); return(options); }
public static ElasticSearchOptions <T> WithIndices <T>(this ElasticSearchOptions <T> options, IEnumerable <string> indices) where T : class { options.Indices.AddRange(indices); return(options); }
public static ElasticSearchOptions <T> WithIndex <T>(this ElasticSearchOptions <T> options, string index) where T : class { options.Indices.Add(index); return(options); }
public static ElasticSearchOptions <T> WithSystemFilter <T>(this ElasticSearchOptions <T> options, string filter) where T : class { options.SystemFilter = !String.IsNullOrEmpty(filter) ? Filter <T> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(filter))) : null; return(options); }
protected ICollection <T> Find(ElasticSearchOptions <T> options) { return(FindAs <T>(options)); }
protected ICollection <TModel> FindAs <TModel>(ElasticSearchOptions <T> options) where TModel : class, new() { if (options == null) { throw new ArgumentNullException("options"); } ICollection <TModel> result; if (EnableCache && options.UseCache) { result = Cache.Get <ICollection <TModel> >(GetScopedCacheKey(options.CacheKey)); if (result != null) { return(result); } } var searchDescriptor = new SearchDescriptor <T>().Filter(options.GetElasticSearchFilter()); searchDescriptor.Indices(options.Indices); searchDescriptor.IgnoreUnavailable(); if (options.UsePaging) { searchDescriptor.Skip(options.GetSkip()); } searchDescriptor.Size(options.GetLimit()); searchDescriptor.Type(typeof(T)); if (options.Fields.Count > 0) { searchDescriptor.Source(s => s.Include(options.Fields.ToArray())); } else { searchDescriptor.Source(s => s.Exclude("idx")); } if (options.SortBy.Count > 0) { foreach (var sort in options.SortBy) { searchDescriptor.Sort(sort); } } _elasticClient.EnableTrace(); var results = _elasticClient.Search <T>(searchDescriptor); _elasticClient.DisableTrace(); if (!results.IsValid) { throw new ApplicationException(String.Format("ElasticSearch error code \"{0}\".", results.ConnectionStatus.HttpStatusCode), results.ConnectionStatus.OriginalException); } options.HasMore = options.UseLimit && results.Total > options.GetLimit(); var items = results.Documents.ToList(); if (typeof(T) != typeof(TModel)) { if (Mapper.FindTypeMapFor <T, TModel>() == null) { Mapper.CreateMap <T, TModel>(); } result = Enumerable.ToList(items.Select(Mapper.Map <T, TModel>)); } else { result = items as List <TModel>; } if (EnableCache && options.UseCache) { Cache.Set(GetScopedCacheKey(options.CacheKey), result, options.GetCacheExpirationDate()); } return(result); }