protected long Count(ElasticSearchOptions <T> options)
{
if (options == null)
{
throw new ArgumentNullException("options");
}
long?result;
if (EnableCache && options.UseCache)
{
result = Cache.Get <long?>(GetScopedCacheKey("count-" + options.CacheKey));
if (result.HasValue)
{
return(result.Value);
}
}
var countDescriptor = new CountDescriptor <T>().Query(f => f.Filtered(s => s.Filter(f2 => options.GetElasticSearchFilter())));
countDescriptor.Indices(options.Indices);
countDescriptor.IgnoreUnavailable();
countDescriptor.Type(typeof(T));
_elasticClient.EnableTrace();
var results = _elasticClient.Count <T>(countDescriptor);
_elasticClient.DisableTrace();
if (!results.IsValid)
{
throw new ApplicationException(String.Format("ElasticSearch error code \"{0}\".", results.ConnectionStatus.HttpStatusCode), results.ConnectionStatus.OriginalException);
}
result = results.Count;
if (EnableCache && options.UseCache)
{
Cache.Set(GetScopedCacheKey("count-" + options.CacheKey), result, options.GetCacheExpirationDate());
}
return(result.Value);
}
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string query = null, TimeSpan?utcOffset = null, int max = 25, int desiredDataPoints = 10)
{
if (!utcOffset.HasValue)
{
utcOffset = TimeSpan.Zero;
}
var allowedTerms = new[] { "tags", "stack_id", "organization_id" };
if (!allowedTerms.Contains(term))
{
throw new ArgumentException("Must be a valid term.", "term");
}
var options = new ElasticSearchOptions <PersistentEvent>().WithQuery(query).WithDateRange(utcStart, utcEnd, "date").WithIndicesFromDateRange();
_client.EnableTrace();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = _client.Search <PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(String.Join(",", options.Indices))
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => options.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.Terms("terms", t => t
.Field(term)
.Size(max)
.Aggregations(agg2 => agg2
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(utcOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
)
)
)
)
);
if (!res.IsValid)
{
Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
_client.DisableTrace();
var stats = new EventTermStatsResult {
Total = res.Aggs.Filter("filtered").DocCount,
};
stats.Terms.AddRange(res.Aggs.Filter("filtered").DateHistogram("terms").Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("unique").Value;
if (timelineUnique.HasValue)
{
count = (long)timelineUnique.Value;
}
var item = new TermStatsItem {
Total = i.DocCount,
Unique = count,
New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0
};
var firstOccurrence = i.Min("first_occurrence").Value;
var lastOccurrence = i.Max("last_occurrence").Value;
if (firstOccurrence.HasValue)
{
item.FirstOccurrence = firstOccurrence.Value.ToDateTime();
}
if (lastOccurrence.HasValue)
{
item.LastOccurrence = lastOccurrence.Value.ToDateTime();
}
item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem {
Date = ti.Date,
Total = ti.DocCount
}));
return(item);
}));
stats.Start = utcStart.Add(utcOffset.Value);
stats.End = utcEnd.Add(utcOffset.Value);
return(stats);
}
public async Task <EventTermStatsResult> GetTermsStatsAsync(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int max = 25, int desiredDataPoints = 10)
{
if (!displayTimeOffset.HasValue)
{
displayTimeOffset = TimeSpan.Zero;
}
var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };
if (!allowedTerms.Contains(term))
{
throw new ArgumentException("Must be a valid term.", nameof(term));
}
var filter = new ElasticSearchOptions <PersistentEvent>()
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithDateRange(utcStart, utcEnd, "date")
.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM");
// if no start date then figure out first event date
if (!filter.UseStartDate)
{
// TODO: Cache this to save an extra search request when a date range isn't filtered.
_elasticClient.EnableTrace();
var result = await _elasticClient.SearchAsync <PersistentEvent>(s => s
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name)
.Filter(d => filter.GetElasticSearchFilter())
.SortAscending(ev => ev.Date)
.Take(1)).AnyContext();
_elasticClient.DisableTrace();
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null)
{
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.WithDateRange(utcStart, utcEnd, "date");
filter.WithIndicesFromDateRange($"'{_eventIndex.VersionedName}-'yyyyMM");
}
}
utcStart = filter.GetStartDate();
utcEnd = filter.GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
_elasticClient.EnableTrace();
var res = await _elasticClient.SearchAsync <PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : _eventIndex.Name)
.Query(filter.GetElasticSearchQuery())
.Aggregations(agg => agg
.Terms("terms", t => t
.Field(term)
.Size(max)
.Aggregations(agg2 => agg2
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(100)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
).AnyContext();
_elasticClient.DisableTrace();
if (!res.IsValid)
{
Logger.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
var newTerms = res.Aggs.Terms("new");
var stats = new EventTermStatsResult {
Total = res.Total,
New = newTerms != null && newTerms.Items.Count > 0 ? newTerms.Items[0].DocCount : 0,
Start = utcStart.SafeAdd(displayTimeOffset.Value),
End = utcEnd.SafeAdd(displayTimeOffset.Value)
};
var unique = res.Aggs.Cardinality("unique");
if (unique?.Value != null)
{
stats.Unique = (long)unique.Value;
}
var firstOccurrence = res.Aggs.Min("first_occurrence");
if (firstOccurrence?.Value != null)
{
stats.FirstOccurrence = firstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
var lastOccurrence = res.Aggs.Max("last_occurrence");
if (lastOccurrence?.Value != null)
{
stats.LastOccurrence = lastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
var terms = res.Aggs.Terms("terms");
if (terms == null)
{
return(stats);
}
stats.Terms.AddRange(terms.Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("unique");
if (timelineUnique?.Value != null)
{
count = (long)timelineUnique.Value;
}
var termNew = i.Terms("new");
var item = new TermStatsItem {
Total = i.DocCount,
Unique = count,
Term = i.Key,
New = termNew != null && termNew.Items.Count > 0 ? termNew.Items[0].DocCount : 0
};
var termFirstOccurrence = i.Min("first_occurrence");
if (termFirstOccurrence?.Value != null)
{
item.FirstOccurrence = termFirstOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
var termLastOccurrence = i.Max("last_occurrence");
if (termLastOccurrence?.Value != null)
{
item.LastOccurrence = termLastOccurrence.Value.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
var timeLine = i.DateHistogram("timelime");
if (timeLine != null)
{
item.Timeline.AddRange(timeLine.Items.Select(ti => new TermTimelineItem {
Date = ti.Date,
Total = ti.DocCount
}));
}
return(item);
}));
return(stats);
}
protected TModel FindOneAs <TModel>(OneOptions options) where TModel : class, new()
{
if (options == null)
{
throw new ArgumentNullException("options");
}
TModel result = null;
if (EnableCache)
{
if (options.UseCache)
{
result = Cache.Get <TModel>(GetScopedCacheKey(options.CacheKey));
}
if (options.UseCache && result != null)
{
Log.Trace().Message("Cache hit: type={0}", _entityType).Write();
}
else if (options.UseCache)
{
Log.Trace().Message("Cache miss: type={0}", _entityType).Write();
}
if (result != null)
{
return(result);
}
}
var searchDescriptor = new SearchDescriptor <T>().Filter(options.GetElasticSearchFilter <T>()).Size(1);
if (options.Fields.Count > 0)
{
searchDescriptor.Source(s => s.Include(options.Fields.ToArray()));
}
else
{
searchDescriptor.Source(s => s.Exclude("idx"));
}
var elasticSearchOptions = options as ElasticSearchOptions <T>;
if (elasticSearchOptions != null && elasticSearchOptions.SortBy.Count > 0)
{
searchDescriptor.Indices(elasticSearchOptions.Indices);
foreach (var sort in elasticSearchOptions.SortBy)
{
searchDescriptor.Sort(sort);
}
}
_elasticClient.EnableTrace();
var item = _elasticClient.Search <T>(searchDescriptor).Documents.FirstOrDefault();
_elasticClient.DisableTrace();
if (typeof(T) != typeof(TModel))
{
if (Mapper.FindTypeMapFor <T, TModel>() == null)
{
Mapper.CreateMap <T, TModel>();
}
result = Mapper.Map <T, TModel>(item);
}
else
{
result = item as TModel;
}
if (EnableCache && result != null && options.UseCache)
{
Cache.Set(GetScopedCacheKey(options.CacheKey), result, options.GetCacheExpirationDate());
}
return(result);
}
public EventTermStatsResult GetTermsStats(DateTime utcStart, DateTime utcEnd, string term, string systemFilter, string userFilter = null, TimeSpan?displayTimeOffset = null, int max = 25, int desiredDataPoints = 10)
{
if (!displayTimeOffset.HasValue)
{
displayTimeOffset = TimeSpan.Zero;
}
var allowedTerms = new[] { "organization_id", "project_id", "stack_id", "tags", "version" };
if (!allowedTerms.Contains(term))
{
throw new ArgumentException("Must be a valid term.", "term");
}
var filter = new ElasticSearchOptions <PersistentEvent>()
.WithFilter(!String.IsNullOrEmpty(systemFilter) ? Filter <PersistentEvent> .Query(q => q.QueryString(qs => qs.DefaultOperator(Operator.And).Query(systemFilter))) : null)
.WithQuery(userFilter)
.WithDateRange(utcStart, utcEnd, "date")
.WithIndicesFromDateRange();
_client.EnableTrace();
// if no start date then figure out first event date
if (!filter.UseStartDate)
{
var result = _client.Search <PersistentEvent>(s => s.IgnoreUnavailable().Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository <PersistentEvent> .EventsIndexName, "-*")).Filter(d => filter.GetElasticSearchFilter()).SortAscending(ev => ev.Date).Take(1));
var firstEvent = result.Hits.FirstOrDefault();
if (firstEvent != null)
{
utcStart = firstEvent.Source.Date.UtcDateTime;
filter.WithDateRange(utcStart, utcEnd, "date");
filter.WithIndicesFromDateRange();
}
}
utcStart = filter.GetStartDate();
utcEnd = filter.GetEndDate();
var interval = GetInterval(utcStart, utcEnd, desiredDataPoints);
var res = _client.Search <PersistentEvent>(s => s
.SearchType(SearchType.Count)
.IgnoreUnavailable()
.Index(filter.Indices.Count > 0 ? String.Join(",", filter.Indices) : String.Concat(ElasticSearchRepository <PersistentEvent> .EventsIndexName, "-*"))
.Aggregations(agg => agg
.Filter("filtered", f => f
.Filter(d => filter.GetElasticSearchFilter())
.Aggregations(filteredAgg => filteredAgg
.Terms("terms", t => t
.Field(term)
.Size(max)
.Aggregations(agg2 => agg2
.DateHistogram("timelime", tl => tl
.Field(ev => ev.Date)
.MinimumDocumentCount(0)
.Interval(interval.Item1)
.TimeZone(HoursAndMinutes(displayTimeOffset.Value))
)
.Cardinality("unique", u => u
.Field(ev => ev.StackId)
.PrecisionThreshold(1000)
)
.Terms("new", u => u
.Field(ev => ev.IsFirstOccurrence)
.Exclude("F")
)
.Min("first_occurrence", o => o.Field(ev => ev.Date))
.Max("last_occurrence", o => o.Field(ev => ev.Date))
)
)
)
)
)
);
if (!res.IsValid)
{
Log.Error().Message("Retrieving term stats failed: {0}", res.ServerError.Error).Write();
throw new ApplicationException("Retrieving term stats failed.");
}
_client.DisableTrace();
var filtered = res.Aggs.Filter("filtered");
if (filtered == null)
{
return(new EventTermStatsResult());
}
var stats = new EventTermStatsResult {
Total = filtered.DocCount
};
stats.Terms.AddRange(filtered.Terms("terms").Items.Select(i => {
long count = 0;
var timelineUnique = i.Cardinality("unique").Value;
if (timelineUnique.HasValue)
{
count = (long)timelineUnique.Value;
}
var item = new TermStatsItem {
Total = i.DocCount,
Unique = count,
Term = i.Key,
New = i.Terms("new").Items.Count > 0 ? i.Terms("new").Items[0].DocCount : 0
};
var firstOccurrence = i.Min("first_occurrence").Value;
var lastOccurrence = i.Max("last_occurrence").Value;
if (firstOccurrence.HasValue)
{
item.FirstOccurrence = firstOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
if (lastOccurrence.HasValue)
{
item.LastOccurrence = lastOccurrence.Value.ToDateTime().SafeAdd(displayTimeOffset.Value);
}
item.Timeline.AddRange(i.DateHistogram("timelime").Items.Select(ti => new TermTimelineItem {
Date = ti.Date,
Total = ti.DocCount
}));
return(item);
}));
stats.Start = utcStart.SafeAdd(displayTimeOffset.Value);
stats.End = utcEnd.SafeAdd(displayTimeOffset.Value);
return(stats);
}
protected ICollection <TModel> FindAs <TModel>(ElasticSearchOptions <T> options) where TModel : class, new()
{
if (options == null)
{
throw new ArgumentNullException("options");
}
ICollection <TModel> result = null;
if (options.UseCache)
{
result = Cache.Get <ICollection <TModel> >(GetScopedCacheKey(options.CacheKey));
}
if (result != null)
{
return(result);
}
var searchDescriptor = new SearchDescriptor <T>().Filter(options.GetElasticSearchFilter());
searchDescriptor.Indices(options.Indices);
searchDescriptor.IgnoreUnavailable();
if (options.UsePaging)
{
searchDescriptor.Skip(options.GetSkip());
}
searchDescriptor.Size(options.GetLimit());
searchDescriptor.Type(typeof(T));
if (options.Fields.Count > 0)
{
searchDescriptor.Source(s => s.Include(options.Fields.ToArray()));
}
if (options.SortBy.Count > 0)
{
foreach (var sort in options.SortBy)
{
searchDescriptor.Sort(sort);
}
}
_elasticClient.EnableTrace();
var results = _elasticClient.Search <T>(searchDescriptor);
_elasticClient.DisableTrace();
if (!results.IsValid)
{
throw new ApplicationException("Error occurred processing request.");
}
Debug.WriteLine("Results: " + results.Total);
options.HasMore = options.UseLimit && results.Total > options.GetLimit();
var items = results.Documents.ToList();
if (typeof(T) != typeof(TModel))
{
if (Mapper.FindTypeMapFor <T, TModel>() == null)
{
Mapper.CreateMap <T, TModel>();
}
result = items.Select(Mapper.Map <T, TModel>).ToList();
}
else
{
result = items as List <TModel>;
}
if (options.UseCache)
{
Cache.Set(GetScopedCacheKey(options.CacheKey), result, options.GetCacheExpirationDate());
}
return(result);
}
protected async Task <FindResults <T> > FindAsync(ElasticSearchOptions <T> options)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (EnableCache && options.UseCache)
{
var cacheValue = await Cache.GetAsync <FindResults <T> >(GetScopedCacheKey(options.CacheKey)).AnyContext();
#if DEBUG
Logger.Trace().Message("Cache {0}: type={1}", cacheValue.HasValue ? "hit" : "miss", _entityType).Write();
#endif
if (cacheValue.HasValue)
{
return(cacheValue.Value);
}
}
var searchDescriptor = new SearchDescriptor <T>();
searchDescriptor.Query(options.GetElasticSearchQuery(_supportsSoftDeletes));
searchDescriptor.Indices(options.Indices.Any() ? options.Indices.ToArray() : GetIndices());
searchDescriptor.IgnoreUnavailable();
searchDescriptor.Size(options.GetLimit());
searchDescriptor.Type(typeof(T));
if (options.UsePaging)
{
searchDescriptor.Skip(options.GetSkip());
}
if (options.Fields.Count > 0)
{
searchDescriptor.Source(s => s.Include(options.Fields.ToArray()));
}
else
{
searchDescriptor.Source(s => s.Exclude("idx"));
}
if (options.SortBy.Count > 0)
{
foreach (var sort in options.SortBy)
{
searchDescriptor.Sort(sort);
}
}
#if DEBUG
_elasticClient.EnableTrace();
var sw = Stopwatch.StartNew();
#endif
var results = await _elasticClient.SearchAsync <T>(searchDescriptor).AnyContext();
#if DEBUG
sw.Stop();
_elasticClient.DisableTrace();
Logger.Trace().Message($"FindAsync: {sw.ElapsedMilliseconds}ms, Elastic Took {results.ElapsedMilliseconds}ms, Serialization Took {results.ConnectionStatus.Metrics.SerializationTime}ms, Deserialization Took {results.ConnectionStatus.Metrics.DeserializationTime}ms").Write();
#endif
if (!results.IsValid)
{
throw new ApplicationException($"ElasticSearch error code \"{results.ConnectionStatus.HttpStatusCode}\".", results.ConnectionStatus.OriginalException);
}
options.HasMore = options.UseLimit && results.Total > options.GetLimit();
var result = new FindResults <T> {
Documents = results.Documents.ToList(),
Total = results.Total
};
if (EnableCache && options.UseCache)
{
await Cache.SetAsync(GetScopedCacheKey(options.CacheKey), result, options.GetCacheExpirationDate()).AnyContext();
}
return(result);
}
