protected long Count(ElasticSearchOptions <T> options) { if (options == null) { throw new ArgumentNullException("options"); } long?result; if (EnableCache && options.UseCache) { result = Cache.Get <long?>(GetScopedCacheKey("count-" + options.CacheKey)); if (result.HasValue) { return(result.Value); } } var countDescriptor = new CountDescriptor <T>().Query(f => f.Filtered(s => s.Filter(f2 => options.GetElasticSearchFilter()))); countDescriptor.Indices(options.Indices); countDescriptor.IgnoreUnavailable(); countDescriptor.Type(typeof(T)); _elasticClient.EnableTrace(); var results = _elasticClient.Count <T>(countDescriptor); _elasticClient.DisableTrace(); if (!results.IsValid) { throw new ApplicationException(String.Format("ElasticSearch error code \"{0}\".", results.ConnectionStatus.HttpStatusCode), results.ConnectionStatus.OriginalException); } result = results.Count; if (EnableCache && options.UseCache) { Cache.Set(GetScopedCacheKey("count-" + options.CacheKey), result, options.GetCacheExpirationDate()); } return(result.Value); }
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string query = null, TimeSpan?utcOffset = null, int max = 25, int desiredDataPoints = 10) { if (!utcOffset.HasValue) { utcOffset = TimeSpan.Zero; } var allowedTerms = new[] { "tags", "stack_id", "organization_id" }; if (!allowedTerms.Contains(term)) { throw new ArgumentException("Must be a valid term.", "term"); } var options = new ElasticSearchOptions <PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange(); _client.EnableTrace(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = _client.Search <PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(String.Join(",", options.Indices)) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => options.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .Terms("terms", t => t .Field(term) .Size(max) .Aggregations(agg2 => agg2 .DateHistogram("timelime", tl => tl .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(utcOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ) ) ) ) ); if (!res.IsValid) { Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving term stats failed."); } _client.DisableTrace(); var stats = new EventTermStatsResult { Total = res.Aggs.Filter("filtered").DocCount, }; stats.Terms.AddRange(res.Aggs.Filter("filtered").DateHistogram("terms").Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("unique").Value; if (timelineUnique.HasValue) { count = (long)timelineUnique.Value; } var item = new TermStatsItem { Total = i.DocCount, Unique = count, New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0 }; var firstOccurrence = i.Min("first_occurrence").Value; var lastOccurrence = i.Max("last_occurrence").Value; if (firstOccurrence.HasValue) { item.FirstOccurrence = firstOccurrence.Value.ToDateTime(); } if (lastOccurrence.HasValue) { item.LastOccurrence = lastOccurrence.Value.ToDateTime(); } item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem { Date = ti.Date, Total = ti.DocCount })); return(item); })); stats.Start = utcStart.Add(utcOffset.Value); stats.End = utcEnd.Add(utcOffset.Value); return(stats); }
public async Task <EventTermStatsResult> GetTermsStatsAsync(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) { if (!displayTimeOffset.HasValue) { displayTimeOffset = TimeSpan.Zero; } var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" }; if (!allowedTerms.Contains(term)) { throw new ArgumentException("Must be a valid term.", nameof(term)); } var filter = new ElasticSearchOptions <PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM"); // if no start date then figure out first event date if (!filter.UseStartDate) { // TODO: Cache this to save an extra search request when a date range isn't filtered. _elasticClient.EnableTrace(); var result = await _elasticClient.SearchAsync <PersistentEvent>(s => s .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Filter(d => filter.GetElasticSearchFilter()) .SortAscending(ev => ev.Date) .Take(1)).AnyContext(); _elasticClient.DisableTrace(); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM"); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); _elasticClient.EnableTrace(); var res = await _elasticClient.SearchAsync <PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name) .Query(filter.GetElasticSearchQuery()) .Aggregations(agg => agg .Terms("terms", t => t .Field(term) .Size(max) .Aggregations(agg2 => agg2 .DateHistogram("timelime", tl => tl .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(100) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ).AnyContext(); _elasticClient.DisableTrace(); if (!res.IsValid) { Logger.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving term stats failed."); } var newTerms = res.Aggs.Terms("new"); var stats = new EventTermStatsResult { Total = res.Total, New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0, Start = utcStart.SafeAdd(displayTimeOffset.Value), End = utcEnd.SafeAdd(displayTimeOffset.Value) }; var unique = res.Aggs.Cardinality("unique"); if (unique?.Value != null) { stats.Unique = (long)unique.Value; } var firstOccurrence = res.Aggs.Min("first_occurrence"); if (firstOccurrence?.Value != null) { stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } var lastOccurrence = res.Aggs.Max("last_occurrence"); if (lastOccurrence?.Value != null) { stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } var terms = res.Aggs.Terms("terms"); if (terms == null) { return(stats); } stats.Terms.AddRange(terms.Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("unique"); if (timelineUnique?.Value != null) { count = (long)timelineUnique.Value; } var termNew = i.Terms("new"); var item = new TermStatsItem { Total = i.DocCount, Unique = count, Term = i.Key, New = termNew != null && termNew.Items.Count > 0 ? termNew.Items[0].DocCount : 0 }; var termFirstOccurrence = i.Min("first_occurrence"); if (termFirstOccurrence?.Value != null) { item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } var termLastOccurrence = i.Max("last_occurrence"); if (termLastOccurrence?.Value != null) { item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } var timeLine = i.DateHistogram("timelime"); if (timeLine != null) { item.Timeline.AddRange(timeLine.Items.Select(ti => new TermTimelineItem { Date = ti.Date, Total = ti.DocCount })); } return(item); })); return(stats); }
protected TModel FindOneAs <TModel>(OneOptions options) where TModel : class, new() { if (options == null) { throw new ArgumentNullException("options"); } TModel result = null; if (EnableCache) { if (options.UseCache) { result = Cache.Get <TModel>(GetScopedCacheKey(options.CacheKey)); } if (options.UseCache && result != null) { Log.Trace().Message("Cache hit: type={0}", _entityType).Write(); } else if (options.UseCache) { Log.Trace().Message("Cache miss: type={0}", _entityType).Write(); } if (result != null) { return(result); } } var searchDescriptor = new SearchDescriptor <T>().Filter(options.GetElasticSearchFilter <T>()).Size(1); if (options.Fields.Count > 0) { searchDescriptor.Source(s => s.Include(options.Fields.ToArray())); } else { searchDescriptor.Source(s => s.Exclude("idx")); } var elasticSearchOptions = options as ElasticSearchOptions <T>; if (elasticSearchOptions != null && elasticSearchOptions.SortBy.Count > 0) { searchDescriptor.Indices(elasticSearchOptions.Indices); foreach (var sort in elasticSearchOptions.SortBy) { searchDescriptor.Sort(sort); } } _elasticClient.EnableTrace(); var item = _elasticClient.Search <T>(searchDescriptor).Documents.FirstOrDefault(); _elasticClient.DisableTrace(); if (typeof(T) != typeof(TModel)) { if (Mapper.FindTypeMapFor <T, TModel>() == null) { Mapper.CreateMap <T, TModel>(); } result = Mapper.Map <T, TModel>(item); } else { result = item as TModel; } if (EnableCache && result != null && options.UseCache) { Cache.Set(GetScopedCacheKey(options.CacheKey), result, options.GetCacheExpirationDate()); } return(result); }
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int max = 25, int desiredDataPoints = 10) { if (!displayTimeOffset.HasValue) { displayTimeOffset = TimeSpan.Zero; } var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" }; if (!allowedTerms.Contains(term)) { throw new ArgumentException("Must be a valid term.", "term"); } var filter = new ElasticSearchOptions <PersistentEvent>() .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null) .WithQuery(userFilter) .WithDateRange(utcStart, utcEnd, "date") .WithIndicesFromDateRange(); _client.EnableTrace(); // if no start date then figure out first event date if (!filter.UseStartDate) { var result = _client.Search <PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository <PersistentEvent> .EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1)); var firstEvent = result.Hits.FirstOrDefault(); if (firstEvent != null) { utcStart = firstEvent.Source.Date.UtcDateTime; filter.WithDateRange(utcStart, utcEnd, "date"); filter.WithIndicesFromDateRange(); } } utcStart = filter.GetStartDate(); utcEnd = filter.GetEndDate(); var interval = GetInterval(utcStart, utcEnd, desiredDataPoints); var res = _client.Search <PersistentEvent>(s => s .SearchType(SearchType.Count) .IgnoreUnavailable() .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository <PersistentEvent> .EventsIndexName, "-*")) .Aggregations(agg => agg .Filter("filtered", f => f .Filter(d => filter.GetElasticSearchFilter()) .Aggregations(filteredAgg => filteredAgg .Terms("terms", t => t .Field(term) .Size(max) .Aggregations(agg2 => agg2 .DateHistogram("timelime", tl => tl .Field(ev => ev.Date) .MinimumDocumentCount(0) .Interval(interval.Item1) .TimeZone(HoursAndMinutes(displayTimeOffset.Value)) ) .Cardinality("unique", u => u .Field(ev => ev.StackId) .PrecisionThreshold(1000) ) .Terms("new", u => u .Field(ev => ev.IsFirstOccurrence) .Exclude("F") ) .Min("first_occurrence", o => o.Field(ev => ev.Date)) .Max("last_occurrence", o => o.Field(ev => ev.Date)) ) ) ) ) ) ); if (!res.IsValid) { Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write(); throw new ApplicationException("Retrieving term stats failed."); } _client.DisableTrace(); var filtered = res.Aggs.Filter("filtered"); if (filtered == null) { return(new EventTermStatsResult()); } var stats = new EventTermStatsResult { Total = filtered.DocCount }; stats.Terms.AddRange(filtered.Terms("terms").Items.Select(i => { long count = 0; var timelineUnique = i.Cardinality("unique").Value; if (timelineUnique.HasValue) { count = (long)timelineUnique.Value; } var item = new TermStatsItem { Total = i.DocCount, Unique = count, Term = i.Key, New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0 }; var firstOccurrence = i.Min("first_occurrence").Value; var lastOccurrence = i.Max("last_occurrence").Value; if (firstOccurrence.HasValue) { item.FirstOccurrence = firstOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } if (lastOccurrence.HasValue) { item.LastOccurrence = lastOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value); } item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem { Date = ti.Date, Total = ti.DocCount })); return(item); })); stats.Start = utcStart.SafeAdd(displayTimeOffset.Value); stats.End = utcEnd.SafeAdd(displayTimeOffset.Value); return(stats); }
protected ICollection <TModel> FindAs <TModel>(ElasticSearchOptions <T> options) where TModel : class, new() { if (options == null) { throw new ArgumentNullException("options"); } ICollection <TModel> result = null; if (options.UseCache) { result = Cache.Get <ICollection <TModel> >(GetScopedCacheKey(options.CacheKey)); } if (result != null) { return(result); } var searchDescriptor = new SearchDescriptor <T>().Filter(options.GetElasticSearchFilter()); searchDescriptor.Indices(options.Indices); searchDescriptor.IgnoreUnavailable(); if (options.UsePaging) { searchDescriptor.Skip(options.GetSkip()); } searchDescriptor.Size(options.GetLimit()); searchDescriptor.Type(typeof(T)); if (options.Fields.Count > 0) { searchDescriptor.Source(s => s.Include(options.Fields.ToArray())); } if (options.SortBy.Count > 0) { foreach (var sort in options.SortBy) { searchDescriptor.Sort(sort); } } _elasticClient.EnableTrace(); var results = _elasticClient.Search <T>(searchDescriptor); _elasticClient.DisableTrace(); if (!results.IsValid) { throw new ApplicationException("Error occurred processing request."); } Debug.WriteLine("Results: " + results.Total); options.HasMore = options.UseLimit && results.Total > options.GetLimit(); var items = results.Documents.ToList(); if (typeof(T) != typeof(TModel)) { if (Mapper.FindTypeMapFor <T, TModel>() == null) { Mapper.CreateMap <T, TModel>(); } result = items.Select(Mapper.Map <T, TModel>).ToList(); } else { result = items as List <TModel>; } if (options.UseCache) { Cache.Set(GetScopedCacheKey(options.CacheKey), result, options.GetCacheExpirationDate()); } return(result); }
protected async Task <FindResults <T> > FindAsync(ElasticSearchOptions <T> options) { if (options == null) { throw new ArgumentNullException(nameof(options)); } if (EnableCache && options.UseCache) { var cacheValue = await Cache.GetAsync <FindResults <T> >(GetScopedCacheKey(options.CacheKey)).AnyContext(); #if DEBUG Logger.Trace().Message("Cache {0}: type={1}", cacheValue.HasValue ? "hit" : "miss", _entityType).Write(); #endif if (cacheValue.HasValue) { return(cacheValue.Value); } } var searchDescriptor = new SearchDescriptor <T>(); searchDescriptor.Query(options.GetElasticSearchQuery(_supportsSoftDeletes)); searchDescriptor.Indices(options.Indices.Any() ? options.Indices.ToArray() : GetIndices()); searchDescriptor.IgnoreUnavailable(); searchDescriptor.Size(options.GetLimit()); searchDescriptor.Type(typeof(T)); if (options.UsePaging) { searchDescriptor.Skip(options.GetSkip()); } if (options.Fields.Count > 0) { searchDescriptor.Source(s => s.Include(options.Fields.ToArray())); } else { searchDescriptor.Source(s => s.Exclude("idx")); } if (options.SortBy.Count > 0) { foreach (var sort in options.SortBy) { searchDescriptor.Sort(sort); } } #if DEBUG _elasticClient.EnableTrace(); var sw = Stopwatch.StartNew(); #endif var results = await _elasticClient.SearchAsync <T>(searchDescriptor).AnyContext(); #if DEBUG sw.Stop(); _elasticClient.DisableTrace(); Logger.Trace().Message($"FindAsync: {sw.ElapsedMilliseconds}ms, Elastic Took {results.ElapsedMilliseconds}ms, Serialization Took {results.ConnectionStatus.Metrics.SerializationTime}ms, Deserialization Took {results.ConnectionStatus.Metrics.DeserializationTime}ms").Write(); #endif if (!results.IsValid) { throw new ApplicationException($"ElasticSearch error code \"{results.ConnectionStatus.HttpStatusCode}\".", results.ConnectionStatus.OriginalException); } options.HasMore = options.UseLimit && results.Total > options.GetLimit(); var result = new FindResults <T> { Documents = results.Documents.ToList(), Total = results.Total }; if (EnableCache && options.UseCache) { await Cache.SetAsync(GetScopedCacheKey(options.CacheKey), result, options.GetCacheExpirationDate()).AnyContext(); } return(result); }