private static InvertedIndex InitializeInvertedIndex()
{
var elasticIndex = new ElasticIndex(elasticServerUrl, elasticServerPort, indexName);
var invertedIndex = new InvertedIndex(elasticIndex);
return(invertedIndex);
}
public virtual async Task <bool> ExistsAsync(Id id, ICommandOptions options = null)
{
if (String.IsNullOrEmpty(id.Value))
{
return(false);
}
// documents that use soft deletes or have parents without a routing id need to use search for exists
if (!SupportsSoftDeletes && (!HasParent || id.Routing != null))
{
var response = await _client.DocumentExistsAsync(new DocumentPath <T>(id.Value), d => {
d.Index(ElasticIndex.GetIndex(id));
if (id.Routing != null)
{
d.Routing(id.Routing);
}
return(d);
}).AnyContext();
_logger.LogRequest(response, options.GetQueryLogLevel());
return(response.Exists);
}
return(await ExistsAsync(q => q.Id(id), o => options.As <T>()).AnyContext());
}
public async Task <bool> IncrementEventCounterAsync(string organizationId, string projectId, string stackId, DateTime minOccurrenceDateUtc, DateTime maxOccurrenceDateUtc, int count, bool sendNotifications = true)
{
// If total occurrences are zero (stack data was reset), then set first occurrence date
// Only update the LastOccurrence if the new date is greater then the existing date.
const string script = @"
Instant parseDate(def dt) {
if (dt != null) {
try {
return Instant.parse(dt);
} catch(DateTimeParseException e) {}
}
return Instant.MIN;
}
if (ctx._source.total_occurrences == 0 || parseDate(ctx._source.first_occurrence).isAfter(parseDate(params.minOccurrenceDateUtc))) {
ctx._source.first_occurrence = params.minOccurrenceDateUtc;
}
if (parseDate(ctx._source.last_occurrence).isBefore(parseDate(params.maxOccurrenceDateUtc))) {
ctx._source.last_occurrence = params.maxOccurrenceDateUtc;
}
if (parseDate(ctx._source.updated_utc).isBefore(parseDate(params.updatedUtc))) {
ctx._source.updated_utc = params.updatedUtc;
}
ctx._source.total_occurrences += params.count;";
var request = new UpdateRequest <Stack, Stack>(ElasticIndex.GetIndex(stackId), stackId)
{
Script = new InlineScript(script.TrimScript())
{
Params = new Dictionary <string, object>(3)
{
{ "minOccurrenceDateUtc", minOccurrenceDateUtc },
{ "maxOccurrenceDateUtc", maxOccurrenceDateUtc },
{ "count", count },
{ "updatedUtc", SystemClock.UtcNow }
}
}
};
var result = await _client.UpdateAsync(request).AnyContext();
if (!result.IsValid)
{
_logger.LogError(result.OriginalException, "Error occurred incrementing total event occurrences on stack {stack}. Error: {Message}", stackId, result.ServerError?.Error);
return(result.ServerError?.Status == 404);
}
if (IsCacheEnabled)
{
await Cache.RemoveAsync(stackId).AnyContext();
}
if (sendNotifications)
{
await PublishMessageAsync(CreateEntityChanged(ChangeType.Saved, organizationId, projectId, null, stackId), TimeSpan.FromSeconds(1.5)).AnyContext();
}
return(true);
}
public virtual async Task <T> GetByIdAsync(Id id, ICommandOptions options = null)
{
if (String.IsNullOrEmpty(id.Value))
{
return(null);
}
options = ConfigureOptions(options.As <T>());
if (IsCacheEnabled && options.HasCacheKey())
{
throw new ArgumentException("Cache key can't be set when calling GetById");
}
if (IsCacheEnabled && options.ShouldReadCache())
{
var value = await GetCachedFindHit(id).AnyContext();
if (value?.Document != null)
{
_logger.LogTrace("Cache hit: type={EntityType} key={Id}", EntityTypeName, id);
return(ShouldReturnDocument(value.Document, options) ? value.Document : null);
}
}
FindHit <T> findHit;
if (!HasParent || id.Routing != null)
{
var request = new GetRequest(ElasticIndex.GetIndex(id), id.Value);
if (id.Routing != null)
{
request.Routing = id.Routing;
}
var response = await _client.GetAsync <T>(request).AnyContext();
_logger.LogRequest(response, options.GetQueryLogLevel());
findHit = response.Found ? response.ToFindHit() : null;
}
else
{
// we don't have the parent id so we have to do a query
// TODO: Ensure this find one query is not cached.
findHit = await FindOneAsync(NewQuery().Id(id), options.Clone().DefaultCacheKey(id)).AnyContext();
}
if (IsCacheEnabled && options.ShouldUseCache())
{
await AddDocumentsToCacheAsync(findHit ?? new FindHit <T>(id, null, 0), options).AnyContext();
}
return(ShouldReturnDocument(findHit?.Document, options) ? findHit?.Document : null);
}
private Task RefreshForConsistency(IRepositoryQuery query, ICommandOptions options)
{
// all docs are saved with immediate or wait consistency, no need to force a refresh
if (DefaultConsistency != Consistency.Eventual)
{
return(Task.CompletedTask);
}
// if using immediate consistency, force a refresh before query
if (options.GetConsistency() == Consistency.Immediate)
{
var indices = ElasticIndex.GetIndexesByQuery(query);
return(_client.Indices.RefreshAsync(indices));
}
return(Task.CompletedTask);
}
protected virtual async Task <SearchDescriptor <T> > ConfigureSearchDescriptorAsync(SearchDescriptor <T> search, IRepositoryQuery query, ICommandOptions options)
{
search ??= new SearchDescriptor <T>();
query = ConfigureQuery(query.As <T>()).Unwrap();
var indices = ElasticIndex.GetIndexesByQuery(query);
if (indices?.Length > 0)
{
search.Index(String.Join(",", indices));
}
if (HasVersion)
{
search.SequenceNumberPrimaryTerm(HasVersion);
}
search.IgnoreUnavailable();
search.TrackTotalHits();
await ElasticIndex.QueryBuilder.ConfigureSearchAsync(query, options, search).AnyContext();
return(search);
}
public SearchService(ElasticIndex index)
{
this.index = index;
}
public ElasticClient GetElasticClient(ElasticIndex index)
{
return(new ElasticClient(_factory, _registry, index.Name, index.Urls, DataOptions));
}
protected string GetElasticKey(ElasticIndex index)
{
return(GetElasticKey(index.Language, index.State));
}
public InvertedIndex(ElasticIndex index)
{
this.index = index;
this.index.CreateIndex();
}
private void SetIndex(string indexName, string serverUri)
{
index = new ElasticIndexTextDocumentIndex(indexName, serverUri);
}
public virtual async Task <IReadOnlyCollection <T> > GetByIdsAsync(Ids ids, ICommandOptions options = null)
{
var idList = ids?.Distinct().Where(i => !String.IsNullOrEmpty(i)).ToList();
if (idList == null || idList.Count == 0)
{
return(EmptyList);
}
if (!HasIdentity)
{
throw new NotSupportedException("Model type must implement IIdentity.");
}
options = ConfigureOptions(options.As <T>());
if (IsCacheEnabled && options.HasCacheKey())
{
throw new ArgumentException("Cache key can't be set when calling GetByIds");
}
var hits = new List <FindHit <T> >();
if (IsCacheEnabled && options.ShouldReadCache())
{
hits.AddRange(await GetCachedFindHit(idList).AnyContext());
}
var itemsToFind = idList.Except(hits.Select(i => (Id)i.Id)).ToList();
if (itemsToFind.Count == 0)
{
return(hits.Where(h => h.Document != null && ShouldReturnDocument(h.Document, options)).Select(h => h.Document).ToList().AsReadOnly());
}
var multiGet = new MultiGetDescriptor();
foreach (var id in itemsToFind.Where(i => i.Routing != null || !HasParent))
{
multiGet.Get <T>(f => {
f.Id(id.Value).Index(ElasticIndex.GetIndex(id));
if (id.Routing != null)
{
f.Routing(id.Routing);
}
return(f);
});
}
var multiGetResults = await _client.MultiGetAsync(multiGet).AnyContext();
_logger.LogRequest(multiGetResults, options.GetQueryLogLevel());
foreach (var doc in multiGetResults.Hits)
{
hits.Add(((IMultiGetHit <T>)doc).ToFindHit());
itemsToFind.Remove(new Id(doc.Id, doc.Routing));
}
// fallback to doing a find
if (itemsToFind.Count > 0 && (HasParent || ElasticIndex.HasMultipleIndexes))
{
var response = await FindAsync(q => q.Id(itemsToFind.Select(id => id.Value)), o => o.PageLimit(1000)).AnyContext();
do
{
if (response.Hits.Count > 0)
{
hits.AddRange(response.Hits.Where(h => h.Document != null));
}
} while (await response.NextPageAsync().AnyContext());
}
if (IsCacheEnabled && options.ShouldUseCache())
{
await AddDocumentsToCacheAsync(hits, options).AnyContext();
}
return(hits.Where(h => h.Document != null && ShouldReturnDocument(h.Document, options)).Select(h => h.Document).ToList().AsReadOnly());
}