public ElasticMultiGetResponse ExecuteMultiGetRequest(MultiGetDescriptor multiGetDescriptor)
{
try
{
var client = GetElasticClient();
var response = client.MultiGet(m => multiGetDescriptor);
return(response.ApiCall.Success
? ElasticMultiGetResponse.SuccessResponse(response)
: ElasticMultiGetResponse.FailResponse(response.ApiCall.OriginalException.Message));
}
catch
{
return(ElasticMultiGetResponse.FailResponse(ServerErrorMessage));
}
}
public MultiGetDescriptor Reduce(MultiGetDescriptor descriptor,
IGrouping <string, object> group)
{
// Validate parameters.
if (descriptor == null)
{
throw new ArgumentNullException(nameof(descriptor));
}
if (group == null)
{
throw new ArgumentNullException(nameof(group));
}
// Get the IDs.
IEnumerable <Id> ids = group.Select(id => id.ToId());
// Aggregate and return.
return(descriptor.GetMany <T>(ids, (d, id) => d.Index(group.Key)));
}
public ElasticResult <T[]> GetAllByGuids <T>(string esType, params string[] guids) where T : class
{
if (guids.Length == 0)
{
return(ElasticResult <T[]> .SuccessResult(new T[] {}));
}
var multiGetDescriptor =
new MultiGetDescriptor().GetMany <T>(guids).Index(_elasticRepository.EsIndex).Type(esType);
var response = _elasticRepository.ExecuteMultiGetRequest(multiGetDescriptor);
if (!response.Success)
{
return(ElasticResult <T[]> .FailResult(response.Message));
}
var multiGetResponse = response.Response;
var hits = multiGetResponse.GetMany <T>(guids);
return(ElasticResult <T[]> .SuccessResult(hits.Select(hit => hit.Source).Where(hit => hit != null).ToArray()));
}
public async Task <IReadOnlyCollection <T> > GetByIdsAsync(Ids ids, ICommandOptions options = null)
{
var idList = ids?.Distinct().Where(i => !String.IsNullOrEmpty(i)).ToList();
if (idList == null || idList.Count == 0)
{
return(EmptyList);
}
if (!HasIdentity)
{
throw new NotSupportedException("Model type must implement IIdentity.");
}
var hits = new List <T>();
if (IsCacheEnabled && options.ShouldReadCache())
{
var cacheHits = await Cache.GetAllAsync <T>(idList.Select(id => id.Value)).AnyContext();
hits.AddRange(cacheHits.Where(kvp => kvp.Value.HasValue).Select(kvp => kvp.Value.Value));
}
var itemsToFind = idList.Except(hits.OfType <IIdentity>().Select(i => (Id)i.Id)).ToList();
if (itemsToFind.Count == 0)
{
return(hits.AsReadOnly());
}
var multiGet = new MultiGetDescriptor();
foreach (var id in itemsToFind.Where(i => i.Routing != null || !HasParent))
{
multiGet.Get <T>(f => {
f.Id(id.Value).Index(GetIndexById(id)).Type(ElasticType.Name);
if (id.Routing != null)
{
f.Routing(id.Routing);
}
return(f);
});
}
var multiGetResults = await _client.MultiGetAsync(multiGet).AnyContext();
_logger.Trace(() => multiGetResults.GetRequest());
foreach (var doc in multiGetResults.Documents)
{
if (!doc.Found)
{
continue;
}
hits.Add(((IMultiGetHit <T>)doc).ToFindHit().Document);
itemsToFind.Remove(doc.Id);
}
// fallback to doing a find
if (itemsToFind.Count > 0 && (HasParent || HasMultipleIndexes))
{
var response = await FindAsync(q => q.Id(itemsToFind.Select(id => id.Value)), o => o.PageLimit(1000)).AnyContext();
do
{
if (response.Hits.Count > 0)
{
hits.AddRange(response.Hits.Where(h => h.Document != null).Select(h => h.Document));
}
} while (await response.NextPageAsync().AnyContext());
}
if (IsCacheEnabled && options.ShouldUseCache())
{
foreach (var item in hits.OfType <IIdentity>())
{
await Cache.SetAsync(item.Id, item, options.GetExpiresIn()).AnyContext();
}
}
return(hits.AsReadOnly());
}
public ICollection <T> GetByIds(ICollection <string> ids, PagingOptions paging = null, bool useCache = false, TimeSpan?expiresIn = null)
{
if (ids == null || ids.Count == 0)
{
return(new List <T>());
}
var results = new List <T>();
if (EnableCache && useCache)
{
results.AddRange(ids.Select(id => Cache.Get <T>(GetScopedCacheKey(id))).Where(cacheHit => cacheHit != null));
var notCachedIds = ids.Except(results.Select(i => i.Id)).ToArray();
if (notCachedIds.Length == 0)
{
return(results);
}
}
// try using the object id to figure out what index the entity is located in
var foundItems = new List <T>();
var itemsToFind = new List <string>();
var multiGet = new MultiGetDescriptor();
foreach (var id in ids.Except(results.Select(i => i.Id)))
{
string index = GetIndexName(id);
if (index != null)
{
multiGet.Get <T>(f => f.Id(id).Index(index).Source(s => s.Exclude("idx")));
}
else
{
itemsToFind.Add(id);
}
}
_elasticClient.EnableTrace();
foreach (var doc in _elasticClient.MultiGet(multiGet).Documents)
{
if (doc.Found)
{
foundItems.Add(doc.Source as T);
}
else
{
itemsToFind.Add(doc.Id);
}
}
_elasticClient.DisableTrace();
// fallback to doing a find
if (itemsToFind.Count > 0)
{
foundItems.AddRange(Find(new ElasticSearchOptions <T>().WithIds(itemsToFind)));
}
if (EnableCache && useCache && foundItems.Count > 0)
{
foreach (var item in foundItems)
{
Cache.Set(GetScopedCacheKey(item.Id), item, expiresIn.HasValue ? DateTime.Now.Add(expiresIn.Value) : DateTime.Now.AddSeconds(RepositoryConstants.DEFAULT_CACHE_EXPIRATION_SECONDS));
}
}
results.AddRange(foundItems);
return(results);
}
public async Task <IReadOnlyCollection <T> > GetByIdsAsync(IEnumerable <string> ids, bool useCache = false, TimeSpan?expiresIn = null)
{
var idList = ids?.Distinct().Where(i => !String.IsNullOrEmpty(i)).ToList();
if (idList == null || idList.Count == 0)
{
return(EmptyList);
}
if (!HasIdentity)
{
throw new NotSupportedException("Model type must implement IIdentity.");
}
var hits = new List <T>();
if (IsCacheEnabled && useCache)
{
var cacheHits = await Cache.GetAllAsync <T>(idList).AnyContext();
hits.AddRange(cacheHits.Where(kvp => kvp.Value.HasValue).Select(kvp => kvp.Value.Value));
}
var itemsToFind = idList.Except(hits.OfType <IIdentity>().Select(i => i.Id)).ToList();
if (itemsToFind.Count == 0)
{
return(hits.AsReadOnly());
}
var multiGet = new MultiGetDescriptor();
if (!HasParent)
{
itemsToFind.ForEach(id => multiGet.Get <T>(f => f.Id(id).Index(GetIndexById(id)).Type(ElasticType.Name)));
var multiGetResults = await _client.MultiGetAsync(multiGet).AnyContext();
_logger.Trace(() => multiGetResults.GetRequest());
foreach (var doc in multiGetResults.Documents)
{
if (!doc.Found)
{
continue;
}
hits.Add(((IMultiGetHit <T>)doc).ToFindHit().Document);
itemsToFind.Remove(doc.Id);
}
}
// fallback to doing a find
if (itemsToFind.Count > 0 && (HasParent || HasMultipleIndexes))
{
hits.AddRange((await FindAsync(NewQuery().WithIds(itemsToFind)).AnyContext()).Hits.Where(h => h.Document != null).Select(h => h.Document));
}
if (IsCacheEnabled && useCache)
{
foreach (var item in hits.OfType <IIdentity>())
{
await Cache.SetAsync(item.Id, item, expiresIn.HasValue?SystemClock.UtcNow.Add(expiresIn.Value) : SystemClock.UtcNow.AddSeconds(ElasticType.DefaultCacheExpirationSeconds)).AnyContext();
}
}
return(hits.AsReadOnly());
}
public MultiGetHitConverter(MultiGetDescriptor descriptor)
{
_descriptor = descriptor;
}
private JsonConverter CreateCovariantMultiGetConverter(MultiGetDescriptor descriptor)
{
var multiGetHitConverter = new MultiGetHitConverter(descriptor);
return multiGetHitConverter;
}
public async Task <FindResults <T> > GetByIdsAsync(ICollection <string> ids, bool useCache = false, TimeSpan?expiresIn = null)
{
var results = new FindResults <T>();
if (ids == null || ids.Count == 0)
{
return(results);
}
var options = Options as IQueryOptions;
if (options == null || !options.HasIdentity)
{
throw new NotSupportedException("Model type must implement IIdentity.");
}
if (IsCacheEnabled && useCache)
{
var cacheHits = await Cache.GetAllAsync <T>(ids.Distinct()).AnyContext();
results.Documents.AddRange(cacheHits.Where(kvp => kvp.Value.HasValue).Select(kvp => kvp.Value.Value));
results.Total = results.Documents.Count;
var notCachedIds = ids.Except(results.Documents.Select(i => ((IIdentity)i).Id)).ToArray();
if (notCachedIds.Length == 0)
{
return(results);
}
}
var itemsToFind = new List <string>(ids.Distinct().Except(results.Documents.Select(i => ((IIdentity)i).Id)));
var multiGet = new MultiGetDescriptor();
if (GetParentIdFunc == null)
{
itemsToFind.ForEach(id => multiGet.Get <T>(f => f.Id(id).Index(GetIndexById(id))));
var multiGetResults = await Context.ElasticClient.MultiGetAsync(multiGet).AnyContext();
foreach (var doc in multiGetResults.Documents)
{
if (!doc.Found)
{
continue;
}
results.Documents.Add(doc.Source as T);
itemsToFind.Remove(doc.Id);
}
}
// fallback to doing a find
if (itemsToFind.Count > 0 && (GetParentIdFunc != null || GetDocumentIndexFunc != null))
{
results.Documents.AddRange((await FindAsync(new ElasticQuery().WithIds(itemsToFind)).AnyContext()).Documents);
}
if (IsCacheEnabled && useCache)
{
foreach (var item in results.Documents)
{
await Cache.SetAsync(((IIdentity)item).Id, item, expiresIn.HasValue?DateTime.UtcNow.Add(expiresIn.Value) : DateTime.UtcNow.AddSeconds(RepositoryConstants.DEFAULT_CACHE_EXPIRATION_SECONDS)).AnyContext();
}
}
results.Total = results.Documents.Count;
return(results);
}
public async Task <FindResults <T> > GetByIdsAsync(ICollection <string> ids, PagingOptions paging = null, bool useCache = false, TimeSpan?expiresIn = null)
{
if (ids == null || ids.Count == 0)
{
return(new FindResults <T>());
}
ids = ids.Where(id => !String.IsNullOrEmpty(id)).Distinct().ToList();
var results = new List <T>();
if (EnableCache && useCache)
{
foreach (var id in ids)
{
var cacheHit = await Cache.GetAsync <T>(GetScopedCacheKey(id)).AnyContext();
if (cacheHit.HasValue)
{
results.Add(cacheHit.Value);
}
}
var notCachedIds = ids.Except(results.Select(i => i.Id)).ToArray();
if (notCachedIds.Length == 0)
{
return new FindResults <T> {
Documents = results, Total = results.Count
}
}
;
}
// try using the object id to figure out what index the entity is located in
var foundItems = new List <T>();
var itemsToFind = new List <string>();
var multiGet = new MultiGetDescriptor();
// TODO Use the index..
foreach (var id in ids.Except(results.Select(i => i.Id)))
{
string index = GetIndexName(id);
if (index != null)
{
multiGet.Get <T>(f => f.Id(id).Index(index).Source(s => s.Exclude("idx")));
}
else
{
itemsToFind.Add(id);
}
}
#if DEBUG
_elasticClient.EnableTrace();
var sw = Stopwatch.StartNew();
#endif
var multiGetResults = await _elasticClient.MultiGetAsync(multiGet).AnyContext();
#if DEBUG
sw.Stop();
_elasticClient.DisableTrace();
Logger.Trace().Message($"FindAsync: {sw.ElapsedMilliseconds}ms, Serialization Took {multiGetResults.ConnectionStatus.Metrics.SerializationTime}ms, Deserialization Took {multiGetResults.ConnectionStatus.Metrics.DeserializationTime}ms").Write();
#endif
foreach (var doc in multiGetResults.Documents)
{
if (doc.Found)
{
foundItems.Add(doc.Source as T);
}
else
{
itemsToFind.Add(doc.Id);
}
}
// fallback to doing a find
if (itemsToFind.Count > 0)
{
foundItems.AddRange((await FindAsync(new ElasticSearchOptions <T>().WithIds(itemsToFind)).AnyContext()).Documents);
}
if (EnableCache && useCache && foundItems.Count > 0)
{
foreach (var item in foundItems)
{
var expiresAtUtc = expiresIn.HasValue ? DateTime.UtcNow.Add(expiresIn.Value) : DateTime.UtcNow.AddSeconds(RepositoryConstants.DEFAULT_CACHE_EXPIRATION_SECONDS);
await Cache.SetAsync(GetScopedCacheKey(item.Id), item, expiresAtUtc).AnyContext();
}
}
results.AddRange(foundItems);
return(new FindResults <T> {
Documents = results,
Total = results.Count
});
}
public virtual async Task <IReadOnlyCollection <T> > GetByIdsAsync(Ids ids, ICommandOptions options = null)
{
var idList = ids?.Distinct().Where(i => !String.IsNullOrEmpty(i)).ToList();
if (idList == null || idList.Count == 0)
{
return(EmptyList);
}
if (!HasIdentity)
{
throw new NotSupportedException("Model type must implement IIdentity.");
}
options = ConfigureOptions(options.As <T>());
if (IsCacheEnabled && options.HasCacheKey())
{
throw new ArgumentException("Cache key can't be set when calling GetByIds");
}
var hits = new List <FindHit <T> >();
if (IsCacheEnabled && options.ShouldReadCache())
{
hits.AddRange(await GetCachedFindHit(idList).AnyContext());
}
var itemsToFind = idList.Except(hits.Select(i => (Id)i.Id)).ToList();
if (itemsToFind.Count == 0)
{
return(hits.Where(h => h.Document != null && ShouldReturnDocument(h.Document, options)).Select(h => h.Document).ToList().AsReadOnly());
}
var multiGet = new MultiGetDescriptor();
foreach (var id in itemsToFind.Where(i => i.Routing != null || !HasParent))
{
multiGet.Get <T>(f => {
f.Id(id.Value).Index(ElasticIndex.GetIndex(id));
if (id.Routing != null)
{
f.Routing(id.Routing);
}
return(f);
});
}
var multiGetResults = await _client.MultiGetAsync(multiGet).AnyContext();
_logger.LogRequest(multiGetResults, options.GetQueryLogLevel());
foreach (var doc in multiGetResults.Hits)
{
hits.Add(((IMultiGetHit <T>)doc).ToFindHit());
itemsToFind.Remove(new Id(doc.Id, doc.Routing));
}
// fallback to doing a find
if (itemsToFind.Count > 0 && (HasParent || ElasticIndex.HasMultipleIndexes))
{
var response = await FindAsync(q => q.Id(itemsToFind.Select(id => id.Value)), o => o.PageLimit(1000)).AnyContext();
do
{
if (response.Hits.Count > 0)
{
hits.AddRange(response.Hits.Where(h => h.Document != null));
}
} while (await response.NextPageAsync().AnyContext());
}
if (IsCacheEnabled && options.ShouldUseCache())
{
await AddDocumentsToCacheAsync(hits, options).AnyContext();
}
return(hits.Where(h => h.Document != null && ShouldReturnDocument(h.Document, options)).Select(h => h.Document).ToList().AsReadOnly());
}