Exemplo n.º 1
0
        protected long Count(ElasticSearchOptions <T> options)
        {
            if (options == null)
            {
                throw new ArgumentNullException("options");
            }

            long?result;

            if (EnableCache && options.UseCache)
            {
                result = Cache.Get <long?>(GetScopedCacheKey("count-" + options.CacheKey));
                if (result.HasValue)
                {
                    return(result.Value);
                }
            }

            var countDescriptor = new CountDescriptor <T>().Query(f => f.Filtered(s => s.Filter(f2 => options.GetElasticSearchFilter())));

            countDescriptor.Indices(options.Indices);
            countDescriptor.IgnoreUnavailable();

            countDescriptor.Type(typeof(T));

            _elasticClient.EnableTrace();
            var results = _elasticClient.Count <T>(countDescriptor);

            _elasticClient.DisableTrace();
            if (!results.IsValid)
            {
                throw new ApplicationException(String.Format("ElasticSearch error code \"{0}\".", results.ConnectionStatus.HttpStatusCode), results.ConnectionStatus.OriginalException);
            }

            result = results.Count;

            if (EnableCache && options.UseCache)
            {
                Cache.Set(GetScopedCacheKey("count-" + options.CacheKey), result, options.GetCacheExpirationDate());
            }

            return(result.Value);
        }
Exemplo n.º 2
0
        public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string query = null, TimeSpan?utcOffset = null, int max = 25, int desiredDataPoints = 10)
        {
            if (!utcOffset.HasValue)
            {
                utcOffset = TimeSpan.Zero;
            }

            var allowedTerms = new[] { "tags", "stack_id", "organization_id" };

            if (!allowedTerms.Contains(term))
            {
                throw new ArgumentException("Must be a valid term.", "term");
            }

            var options = new ElasticSearchOptions <PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange();

            _client.EnableTrace();

            var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
            var res      = _client.Search <PersistentEvent>(s => s
                                                            .SearchType(SearchType.Count)
                                                            .IgnoreUnavailable()
                                                            .Index(String.Join(",", options.Indices))
                                                            .Aggregations(agg => agg
                                                                          .Filter("filtered", f => f
                                                                                  .Filter(d => options.GetElasticSearchFilter())
                                                                                  .Aggregations(filteredAgg => filteredAgg
                                                                                                .Terms("terms", t => t
                                                                                                       .Field(term)
                                                                                                       .Size(max)
                                                                                                       .Aggregations(agg2 => agg2
                                                                                                                     .DateHistogram("timelime", tl => tl
                                                                                                                                    .Field(ev => ev.Date)
                                                                                                                                    .MinimumDocumentCount(0)
                                                                                                                                    .Interval(interval.Item1)
                                                                                                                                    .TimeZone(HoursAndMinutes(utcOffset.Value))
                                                                                                                                    )
                                                                                                                     .Cardinality("unique", u => u
                                                                                                                                  .Field(ev => ev.StackId)
                                                                                                                                  .PrecisionThreshold(1000)
                                                                                                                                  )
                                                                                                                     .Terms("new", u => u
                                                                                                                            .Field(ev => ev.IsFirstOccurrence)
                                                                                                                            .Exclude("F")
                                                                                                                            )
                                                                                                                     .Min("first_occurrence", o => o.Field(ev => ev.Date))
                                                                                                                     .Max("last_occurrence", o => o.Field(ev => ev.Date))
                                                                                                                     )
                                                                                                       )
                                                                                                )
                                                                                  )
                                                                          )
                                                            );

            if (!res.IsValid)
            {
                Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
                throw new ApplicationException("Retrieving term stats failed.");
            }

            _client.DisableTrace();

            var stats = new EventTermStatsResult {
                Total = res.Aggs.Filter("filtered").DocCount,
            };

            stats.Terms.AddRange(res.Aggs.Filter("filtered").DateHistogram("terms").Items.Select(i => {
                long count         = 0;
                var timelineUnique = i.Cardinality("unique").Value;
                if (timelineUnique.HasValue)
                {
                    count = (long)timelineUnique.Value;
                }

                var item = new TermStatsItem {
                    Total  = i.DocCount,
                    Unique = count,
                    New    = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0
                };

                var firstOccurrence = i.Min("first_occurrence").Value;
                var lastOccurrence  = i.Max("last_occurrence").Value;

                if (firstOccurrence.HasValue)
                {
                    item.FirstOccurrence = firstOccurrence.Value.ToDateTime();
                }

                if (lastOccurrence.HasValue)
                {
                    item.LastOccurrence = lastOccurrence.Value.ToDateTime();
                }

                item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem {
                    Date  = ti.Date,
                    Total = ti.DocCount
                }));

                return(item);
            }));

            stats.Start = utcStart.Add(utcOffset.Value);
            stats.End   = utcEnd.Add(utcOffset.Value);

            return(stats);
        }
Exemplo n.º 3
0
        public async Task <EventTermStatsResult> GetTermsStatsAsync(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int max = 25, int desiredDataPoints = 10)
        {
            if (!displayTimeOffset.HasValue)
            {
                displayTimeOffset = TimeSpan.Zero;
            }

            var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };

            if (!allowedTerms.Contains(term))
            {
                throw new ArgumentException("Must be a valid term.", nameof(term));
            }

            var filter = new ElasticSearchOptions <PersistentEvent>()
                         .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
                         .WithQuery(userFilter)
                         .WithDateRange(utcStart, utcEnd, "date")
                         .WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM");

            // if no start date then figure out first event date
            if (!filter.UseStartDate)
            {
                // TODO: Cache this to save an extra search request when a date range isn't filtered.
                _elasticClient.EnableTrace();

                var result = await _elasticClient.SearchAsync <PersistentEvent>(s => s
                                                                                .IgnoreUnavailable()
                                                                                .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name)
                                                                                .Filter(d => filter.GetElasticSearchFilter())
                                                                                .SortAscending(ev => ev.Date)
                                                                                .Take(1)).AnyContext();

                _elasticClient.DisableTrace();

                var firstEvent = result.Hits.FirstOrDefault();
                if (firstEvent != null)
                {
                    utcStart = firstEvent.Source.Date.UtcDateTime;
                    filter.WithDateRange(utcStart, utcEnd, "date");
                    filter.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM");
                }
            }

            utcStart = filter.GetStartDate();
            utcEnd   = filter.GetEndDate();
            var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);

            _elasticClient.EnableTrace();
            var res = await _elasticClient.SearchAsync <PersistentEvent>(s => s
                                                                         .SearchType(SearchType.Count)
                                                                         .IgnoreUnavailable()
                                                                         .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name)
                                                                         .Query(filter.GetElasticSearchQuery())
                                                                         .Aggregations(agg => agg
                                                                                       .Terms("terms", t => t
                                                                                              .Field(term)
                                                                                              .Size(max)
                                                                                              .Aggregations(agg2 => agg2
                                                                                                            .DateHistogram("timelime", tl => tl
                                                                                                                           .Field(ev => ev.Date)
                                                                                                                           .MinimumDocumentCount(0)
                                                                                                                           .Interval(interval.Item1)
                                                                                                                           .TimeZone(HoursAndMinutes(displayTimeOffset.Value))
                                                                                                                           )
                                                                                                            .Cardinality("unique", u => u
                                                                                                                         .Field(ev => ev.StackId)
                                                                                                                         .PrecisionThreshold(100)
                                                                                                                         )
                                                                                                            .Terms("new", u => u
                                                                                                                   .Field(ev => ev.IsFirstOccurrence)
                                                                                                                   .Exclude("F")
                                                                                                                   )
                                                                                                            .Min("first_occurrence", o => o.Field(ev => ev.Date))
                                                                                                            .Max("last_occurrence", o => o.Field(ev => ev.Date))
                                                                                                            )
                                                                                              )
                                                                                       .Cardinality("unique", u => u
                                                                                                    .Field(ev => ev.StackId)
                                                                                                    .PrecisionThreshold(100)
                                                                                                    )
                                                                                       .Terms("new", u => u
                                                                                              .Field(ev => ev.IsFirstOccurrence)
                                                                                              .Exclude("F")
                                                                                              )
                                                                                       .Min("first_occurrence", o => o.Field(ev => ev.Date))
                                                                                       .Max("last_occurrence", o => o.Field(ev => ev.Date))
                                                                                       )
                                                                         ).AnyContext();

            _elasticClient.DisableTrace();

            if (!res.IsValid)
            {
                Logger.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
                throw new ApplicationException("Retrieving term stats failed.");
            }

            var newTerms = res.Aggs.Terms("new");
            var stats    = new EventTermStatsResult {
                Total = res.Total,
                New   = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0,
                Start = utcStart.SafeAdd(displayTimeOffset.Value),
                End   = utcEnd.SafeAdd(displayTimeOffset.Value)
            };

            var unique = res.Aggs.Cardinality("unique");

            if (unique?.Value != null)
            {
                stats.Unique = (long)unique.Value;
            }

            var firstOccurrence = res.Aggs.Min("first_occurrence");

            if (firstOccurrence?.Value != null)
            {
                stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
            }

            var lastOccurrence = res.Aggs.Max("last_occurrence");

            if (lastOccurrence?.Value != null)
            {
                stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
            }

            var terms = res.Aggs.Terms("terms");

            if (terms == null)
            {
                return(stats);
            }

            stats.Terms.AddRange(terms.Items.Select(i => {
                long count         = 0;
                var timelineUnique = i.Cardinality("unique");
                if (timelineUnique?.Value != null)
                {
                    count = (long)timelineUnique.Value;
                }

                var termNew = i.Terms("new");
                var item    = new TermStatsItem {
                    Total  = i.DocCount,
                    Unique = count,
                    Term   = i.Key,
                    New    = termNew != null && termNew.Items.Count > 0 ? termNew.Items[0].DocCount : 0
                };

                var termFirstOccurrence = i.Min("first_occurrence");
                if (termFirstOccurrence?.Value != null)
                {
                    item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
                }

                var termLastOccurrence = i.Max("last_occurrence");
                if (termLastOccurrence?.Value != null)
                {
                    item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
                }

                var timeLine = i.DateHistogram("timelime");
                if (timeLine != null)
                {
                    item.Timeline.AddRange(timeLine.Items.Select(ti => new TermTimelineItem {
                        Date  = ti.Date,
                        Total = ti.DocCount
                    }));
                }

                return(item);
            }));

            return(stats);
        }
        protected TModel FindOneAs <TModel>(OneOptions options) where TModel : class, new()
        {
            if (options == null)
            {
                throw new ArgumentNullException("options");
            }

            TModel result = null;

            if (EnableCache)
            {
                if (options.UseCache)
                {
                    result = Cache.Get <TModel>(GetScopedCacheKey(options.CacheKey));
                }

                if (options.UseCache && result != null)
                {
                    Log.Trace().Message("Cache hit: type={0}", _entityType).Write();
                }
                else if (options.UseCache)
                {
                    Log.Trace().Message("Cache miss: type={0}", _entityType).Write();
                }

                if (result != null)
                {
                    return(result);
                }
            }

            var searchDescriptor = new SearchDescriptor <T>().Filter(options.GetElasticSearchFilter <T>()).Size(1);

            if (options.Fields.Count > 0)
            {
                searchDescriptor.Source(s => s.Include(options.Fields.ToArray()));
            }
            else
            {
                searchDescriptor.Source(s => s.Exclude("idx"));
            }

            var elasticSearchOptions = options as ElasticSearchOptions <T>;

            if (elasticSearchOptions != null && elasticSearchOptions.SortBy.Count > 0)
            {
                searchDescriptor.Indices(elasticSearchOptions.Indices);
                foreach (var sort in elasticSearchOptions.SortBy)
                {
                    searchDescriptor.Sort(sort);
                }
            }

            _elasticClient.EnableTrace();
            var item = _elasticClient.Search <T>(searchDescriptor).Documents.FirstOrDefault();

            _elasticClient.DisableTrace();

            if (typeof(T) != typeof(TModel))
            {
                if (Mapper.FindTypeMapFor <T, TModel>() == null)
                {
                    Mapper.CreateMap <T, TModel>();
                }

                result = Mapper.Map <T, TModel>(item);
            }
            else
            {
                result = item as TModel;
            }

            if (EnableCache && result != null && options.UseCache)
            {
                Cache.Set(GetScopedCacheKey(options.CacheKey), result, options.GetCacheExpirationDate());
            }

            return(result);
        }
Exemplo n.º 5
0
        public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int max = 25, int desiredDataPoints = 10)
        {
            if (!displayTimeOffset.HasValue)
            {
                displayTimeOffset = TimeSpan.Zero;
            }

            var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };

            if (!allowedTerms.Contains(term))
            {
                throw new ArgumentException("Must be a valid term.", "term");
            }

            var filter = new ElasticSearchOptions <PersistentEvent>()
                         .WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
                         .WithQuery(userFilter)
                         .WithDateRange(utcStart, utcEnd, "date")
                         .WithIndicesFromDateRange();

            _client.EnableTrace();

            // if no start date then figure out first event date
            if (!filter.UseStartDate)
            {
                var result = _client.Search <PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository <PersistentEvent> .EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1));

                var firstEvent = result.Hits.FirstOrDefault();
                if (firstEvent != null)
                {
                    utcStart = firstEvent.Source.Date.UtcDateTime;
                    filter.WithDateRange(utcStart, utcEnd, "date");
                    filter.WithIndicesFromDateRange();
                }
            }

            utcStart = filter.GetStartDate();
            utcEnd   = filter.GetEndDate();

            var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
            var res      = _client.Search <PersistentEvent>(s => s
                                                            .SearchType(SearchType.Count)
                                                            .IgnoreUnavailable()
                                                            .Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository <PersistentEvent> .EventsIndexName, "-*"))
                                                            .Aggregations(agg => agg
                                                                          .Filter("filtered", f => f
                                                                                  .Filter(d => filter.GetElasticSearchFilter())
                                                                                  .Aggregations(filteredAgg => filteredAgg
                                                                                                .Terms("terms", t => t
                                                                                                       .Field(term)
                                                                                                       .Size(max)
                                                                                                       .Aggregations(agg2 => agg2
                                                                                                                     .DateHistogram("timelime", tl => tl
                                                                                                                                    .Field(ev => ev.Date)
                                                                                                                                    .MinimumDocumentCount(0)
                                                                                                                                    .Interval(interval.Item1)
                                                                                                                                    .TimeZone(HoursAndMinutes(displayTimeOffset.Value))
                                                                                                                                    )
                                                                                                                     .Cardinality("unique", u => u
                                                                                                                                  .Field(ev => ev.StackId)
                                                                                                                                  .PrecisionThreshold(1000)
                                                                                                                                  )
                                                                                                                     .Terms("new", u => u
                                                                                                                            .Field(ev => ev.IsFirstOccurrence)
                                                                                                                            .Exclude("F")
                                                                                                                            )
                                                                                                                     .Min("first_occurrence", o => o.Field(ev => ev.Date))
                                                                                                                     .Max("last_occurrence", o => o.Field(ev => ev.Date))
                                                                                                                     )
                                                                                                       )
                                                                                                )
                                                                                  )
                                                                          )
                                                            );

            if (!res.IsValid)
            {
                Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
                throw new ApplicationException("Retrieving term stats failed.");
            }

            _client.DisableTrace();


            var filtered = res.Aggs.Filter("filtered");

            if (filtered == null)
            {
                return(new EventTermStatsResult());
            }

            var stats = new EventTermStatsResult {
                Total = filtered.DocCount
            };

            stats.Terms.AddRange(filtered.Terms("terms").Items.Select(i => {
                long count         = 0;
                var timelineUnique = i.Cardinality("unique").Value;
                if (timelineUnique.HasValue)
                {
                    count = (long)timelineUnique.Value;
                }

                var item = new TermStatsItem {
                    Total  = i.DocCount,
                    Unique = count,
                    Term   = i.Key,
                    New    = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0
                };

                var firstOccurrence = i.Min("first_occurrence").Value;
                var lastOccurrence  = i.Max("last_occurrence").Value;

                if (firstOccurrence.HasValue)
                {
                    item.FirstOccurrence = firstOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
                }

                if (lastOccurrence.HasValue)
                {
                    item.LastOccurrence = lastOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
                }

                item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem {
                    Date  = ti.Date,
                    Total = ti.DocCount
                }));

                return(item);
            }));

            stats.Start = utcStart.SafeAdd(displayTimeOffset.Value);
            stats.End   = utcEnd.SafeAdd(displayTimeOffset.Value);

            return(stats);
        }
Exemplo n.º 6
0
        protected ICollection <TModel> FindAs <TModel>(ElasticSearchOptions <T> options) where TModel : class, new()
        {
            if (options == null)
            {
                throw new ArgumentNullException("options");
            }

            ICollection <TModel> result = null;

            if (options.UseCache)
            {
                result = Cache.Get <ICollection <TModel> >(GetScopedCacheKey(options.CacheKey));
            }

            if (result != null)
            {
                return(result);
            }

            var searchDescriptor = new SearchDescriptor <T>().Filter(options.GetElasticSearchFilter());

            searchDescriptor.Indices(options.Indices);
            searchDescriptor.IgnoreUnavailable();

            if (options.UsePaging)
            {
                searchDescriptor.Skip(options.GetSkip());
            }
            searchDescriptor.Size(options.GetLimit());
            searchDescriptor.Type(typeof(T));
            if (options.Fields.Count > 0)
            {
                searchDescriptor.Source(s => s.Include(options.Fields.ToArray()));
            }
            if (options.SortBy.Count > 0)
            {
                foreach (var sort in options.SortBy)
                {
                    searchDescriptor.Sort(sort);
                }
            }

            _elasticClient.EnableTrace();
            var results = _elasticClient.Search <T>(searchDescriptor);

            _elasticClient.DisableTrace();
            if (!results.IsValid)
            {
                throw new ApplicationException("Error occurred processing request.");
            }

            Debug.WriteLine("Results: " + results.Total);
            options.HasMore = options.UseLimit && results.Total > options.GetLimit();

            var items = results.Documents.ToList();

            if (typeof(T) != typeof(TModel))
            {
                if (Mapper.FindTypeMapFor <T, TModel>() == null)
                {
                    Mapper.CreateMap <T, TModel>();
                }

                result = items.Select(Mapper.Map <T, TModel>).ToList();
            }
            else
            {
                result = items as List <TModel>;
            }

            if (options.UseCache)
            {
                Cache.Set(GetScopedCacheKey(options.CacheKey), result, options.GetCacheExpirationDate());
            }

            return(result);
        }
        protected async Task <FindResults <T> > FindAsync(ElasticSearchOptions <T> options)
        {
            if (options == null)
            {
                throw new ArgumentNullException(nameof(options));
            }

            if (EnableCache && options.UseCache)
            {
                var cacheValue = await Cache.GetAsync <FindResults <T> >(GetScopedCacheKey(options.CacheKey)).AnyContext();

#if DEBUG
                Logger.Trace().Message("Cache {0}: type={1}", cacheValue.HasValue ? "hit" : "miss", _entityType).Write();
#endif
                if (cacheValue.HasValue)
                {
                    return(cacheValue.Value);
                }
            }

            var searchDescriptor = new SearchDescriptor <T>();
            searchDescriptor.Query(options.GetElasticSearchQuery(_supportsSoftDeletes));
            searchDescriptor.Indices(options.Indices.Any() ? options.Indices.ToArray() : GetIndices());
            searchDescriptor.IgnoreUnavailable();
            searchDescriptor.Size(options.GetLimit());
            searchDescriptor.Type(typeof(T));

            if (options.UsePaging)
            {
                searchDescriptor.Skip(options.GetSkip());
            }

            if (options.Fields.Count > 0)
            {
                searchDescriptor.Source(s => s.Include(options.Fields.ToArray()));
            }
            else
            {
                searchDescriptor.Source(s => s.Exclude("idx"));
            }

            if (options.SortBy.Count > 0)
            {
                foreach (var sort in options.SortBy)
                {
                    searchDescriptor.Sort(sort);
                }
            }
#if DEBUG
            _elasticClient.EnableTrace();
            var sw = Stopwatch.StartNew();
#endif
            var results = await _elasticClient.SearchAsync <T>(searchDescriptor).AnyContext();

#if DEBUG
            sw.Stop();
            _elasticClient.DisableTrace();
            Logger.Trace().Message($"FindAsync: {sw.ElapsedMilliseconds}ms, Elastic Took {results.ElapsedMilliseconds}ms, Serialization Took {results.ConnectionStatus.Metrics.SerializationTime}ms, Deserialization Took {results.ConnectionStatus.Metrics.DeserializationTime}ms").Write();
#endif

            if (!results.IsValid)
            {
                throw new ApplicationException($"ElasticSearch error code \"{results.ConnectionStatus.HttpStatusCode}\".", results.ConnectionStatus.OriginalException);
            }

            options.HasMore = options.UseLimit && results.Total > options.GetLimit();

            var result = new FindResults <T> {
                Documents = results.Documents.ToList(),
                Total     = results.Total
            };

            if (EnableCache && options.UseCache)
            {
                await Cache.SetAsync(GetScopedCacheKey(options.CacheKey), result, options.GetCacheExpirationDate()).AnyContext();
            }

            return(result);
        }