Esempio n. 1
0
        public ElasticMultiGetResponse ExecuteMultiGetRequest(MultiGetDescriptor multiGetDescriptor)
        {
            try
            {
                var client   = GetElasticClient();
                var response = client.MultiGet(m => multiGetDescriptor);

                return(response.ApiCall.Success
                    ? ElasticMultiGetResponse.SuccessResponse(response)
                    : ElasticMultiGetResponse.FailResponse(response.ApiCall.OriginalException.Message));
            }
            catch
            {
                return(ElasticMultiGetResponse.FailResponse(ServerErrorMessage));
            }
        }
            public MultiGetDescriptor Reduce(MultiGetDescriptor descriptor,
                                             IGrouping <string, object> group)
            {
                // Validate parameters.
                if (descriptor == null)
                {
                    throw new ArgumentNullException(nameof(descriptor));
                }
                if (group == null)
                {
                    throw new ArgumentNullException(nameof(group));
                }

                // Get the IDs.
                IEnumerable <Id> ids = group.Select(id => id.ToId());

                // Aggregate and return.
                return(descriptor.GetMany <T>(ids, (d, id) => d.Index(group.Key)));
            }
Esempio n. 3
0
        public ElasticResult <T[]> GetAllByGuids <T>(string esType, params string[] guids) where T : class
        {
            if (guids.Length == 0)
            {
                return(ElasticResult <T[]> .SuccessResult(new T[] {}));
            }

            var multiGetDescriptor =
                new MultiGetDescriptor().GetMany <T>(guids).Index(_elasticRepository.EsIndex).Type(esType);

            var response = _elasticRepository.ExecuteMultiGetRequest(multiGetDescriptor);

            if (!response.Success)
            {
                return(ElasticResult <T[]> .FailResult(response.Message));
            }

            var multiGetResponse = response.Response;
            var hits             = multiGetResponse.GetMany <T>(guids);

            return(ElasticResult <T[]> .SuccessResult(hits.Select(hit => hit.Source).Where(hit => hit != null).ToArray()));
        }
        public async Task <IReadOnlyCollection <T> > GetByIdsAsync(Ids ids, ICommandOptions options = null)
        {
            var idList = ids?.Distinct().Where(i => !String.IsNullOrEmpty(i)).ToList();

            if (idList == null || idList.Count == 0)
            {
                return(EmptyList);
            }

            if (!HasIdentity)
            {
                throw new NotSupportedException("Model type must implement IIdentity.");
            }

            var hits = new List <T>();

            if (IsCacheEnabled && options.ShouldReadCache())
            {
                var cacheHits = await Cache.GetAllAsync <T>(idList.Select(id => id.Value)).AnyContext();

                hits.AddRange(cacheHits.Where(kvp => kvp.Value.HasValue).Select(kvp => kvp.Value.Value));
            }

            var itemsToFind = idList.Except(hits.OfType <IIdentity>().Select(i => (Id)i.Id)).ToList();

            if (itemsToFind.Count == 0)
            {
                return(hits.AsReadOnly());
            }

            var multiGet = new MultiGetDescriptor();

            foreach (var id in itemsToFind.Where(i => i.Routing != null || !HasParent))
            {
                multiGet.Get <T>(f => {
                    f.Id(id.Value).Index(GetIndexById(id)).Type(ElasticType.Name);
                    if (id.Routing != null)
                    {
                        f.Routing(id.Routing);
                    }

                    return(f);
                });
            }

            var multiGetResults = await _client.MultiGetAsync(multiGet).AnyContext();

            _logger.Trace(() => multiGetResults.GetRequest());

            foreach (var doc in multiGetResults.Documents)
            {
                if (!doc.Found)
                {
                    continue;
                }

                hits.Add(((IMultiGetHit <T>)doc).ToFindHit().Document);
                itemsToFind.Remove(doc.Id);
            }

            // fallback to doing a find
            if (itemsToFind.Count > 0 && (HasParent || HasMultipleIndexes))
            {
                var response = await FindAsync(q => q.Id(itemsToFind.Select(id => id.Value)), o => o.PageLimit(1000)).AnyContext();

                do
                {
                    if (response.Hits.Count > 0)
                    {
                        hits.AddRange(response.Hits.Where(h => h.Document != null).Select(h => h.Document));
                    }
                } while (await response.NextPageAsync().AnyContext());
            }

            if (IsCacheEnabled && options.ShouldUseCache())
            {
                foreach (var item in hits.OfType <IIdentity>())
                {
                    await Cache.SetAsync(item.Id, item, options.GetExpiresIn()).AnyContext();
                }
            }

            return(hits.AsReadOnly());
        }
        public ICollection <T> GetByIds(ICollection <string> ids, PagingOptions paging = null, bool useCache = false, TimeSpan?expiresIn = null)
        {
            if (ids == null || ids.Count == 0)
            {
                return(new List <T>());
            }

            var results = new List <T>();

            if (EnableCache && useCache)
            {
                results.AddRange(ids.Select(id => Cache.Get <T>(GetScopedCacheKey(id))).Where(cacheHit => cacheHit != null));

                var notCachedIds = ids.Except(results.Select(i => i.Id)).ToArray();
                if (notCachedIds.Length == 0)
                {
                    return(results);
                }
            }

            // try using the object id to figure out what index the entity is located in
            var foundItems  = new List <T>();
            var itemsToFind = new List <string>();
            var multiGet    = new MultiGetDescriptor();

            foreach (var id in ids.Except(results.Select(i => i.Id)))
            {
                string index = GetIndexName(id);
                if (index != null)
                {
                    multiGet.Get <T>(f => f.Id(id).Index(index).Source(s => s.Exclude("idx")));
                }
                else
                {
                    itemsToFind.Add(id);
                }
            }

            _elasticClient.EnableTrace();
            foreach (var doc in _elasticClient.MultiGet(multiGet).Documents)
            {
                if (doc.Found)
                {
                    foundItems.Add(doc.Source as T);
                }
                else
                {
                    itemsToFind.Add(doc.Id);
                }
            }
            _elasticClient.DisableTrace();

            // fallback to doing a find
            if (itemsToFind.Count > 0)
            {
                foundItems.AddRange(Find(new ElasticSearchOptions <T>().WithIds(itemsToFind)));
            }

            if (EnableCache && useCache && foundItems.Count > 0)
            {
                foreach (var item in foundItems)
                {
                    Cache.Set(GetScopedCacheKey(item.Id), item, expiresIn.HasValue ? DateTime.Now.Add(expiresIn.Value) : DateTime.Now.AddSeconds(RepositoryConstants.DEFAULT_CACHE_EXPIRATION_SECONDS));
                }
            }

            results.AddRange(foundItems);
            return(results);
        }
Esempio n. 6
0
        public async Task <IReadOnlyCollection <T> > GetByIdsAsync(IEnumerable <string> ids, bool useCache = false, TimeSpan?expiresIn = null)
        {
            var idList = ids?.Distinct().Where(i => !String.IsNullOrEmpty(i)).ToList();

            if (idList == null || idList.Count == 0)
            {
                return(EmptyList);
            }

            if (!HasIdentity)
            {
                throw new NotSupportedException("Model type must implement IIdentity.");
            }

            var hits = new List <T>();

            if (IsCacheEnabled && useCache)
            {
                var cacheHits = await Cache.GetAllAsync <T>(idList).AnyContext();

                hits.AddRange(cacheHits.Where(kvp => kvp.Value.HasValue).Select(kvp => kvp.Value.Value));
            }

            var itemsToFind = idList.Except(hits.OfType <IIdentity>().Select(i => i.Id)).ToList();

            if (itemsToFind.Count == 0)
            {
                return(hits.AsReadOnly());
            }

            var multiGet = new MultiGetDescriptor();

            if (!HasParent)
            {
                itemsToFind.ForEach(id => multiGet.Get <T>(f => f.Id(id).Index(GetIndexById(id)).Type(ElasticType.Name)));

                var multiGetResults = await _client.MultiGetAsync(multiGet).AnyContext();

                _logger.Trace(() => multiGetResults.GetRequest());

                foreach (var doc in multiGetResults.Documents)
                {
                    if (!doc.Found)
                    {
                        continue;
                    }

                    hits.Add(((IMultiGetHit <T>)doc).ToFindHit().Document);
                    itemsToFind.Remove(doc.Id);
                }
            }

            // fallback to doing a find
            if (itemsToFind.Count > 0 && (HasParent || HasMultipleIndexes))
            {
                hits.AddRange((await FindAsync(NewQuery().WithIds(itemsToFind)).AnyContext()).Hits.Where(h => h.Document != null).Select(h => h.Document));
            }

            if (IsCacheEnabled && useCache)
            {
                foreach (var item in hits.OfType <IIdentity>())
                {
                    await Cache.SetAsync(item.Id, item, expiresIn.HasValue?SystemClock.UtcNow.Add(expiresIn.Value) : SystemClock.UtcNow.AddSeconds(ElasticType.DefaultCacheExpirationSeconds)).AnyContext();
                }
            }

            return(hits.AsReadOnly());
        }
 public MultiGetHitConverter(MultiGetDescriptor descriptor)
 {
     _descriptor = descriptor;
 }
 private JsonConverter CreateCovariantMultiGetConverter(MultiGetDescriptor descriptor)
 {
     var multiGetHitConverter = new MultiGetHitConverter(descriptor);
     return multiGetHitConverter;
 }
        public async Task <FindResults <T> > GetByIdsAsync(ICollection <string> ids, bool useCache = false, TimeSpan?expiresIn = null)
        {
            var results = new FindResults <T>();

            if (ids == null || ids.Count == 0)
            {
                return(results);
            }

            var options = Options as IQueryOptions;

            if (options == null || !options.HasIdentity)
            {
                throw new NotSupportedException("Model type must implement IIdentity.");
            }

            if (IsCacheEnabled && useCache)
            {
                var cacheHits = await Cache.GetAllAsync <T>(ids.Distinct()).AnyContext();

                results.Documents.AddRange(cacheHits.Where(kvp => kvp.Value.HasValue).Select(kvp => kvp.Value.Value));
                results.Total = results.Documents.Count;

                var notCachedIds = ids.Except(results.Documents.Select(i => ((IIdentity)i).Id)).ToArray();
                if (notCachedIds.Length == 0)
                {
                    return(results);
                }
            }

            var itemsToFind = new List <string>(ids.Distinct().Except(results.Documents.Select(i => ((IIdentity)i).Id)));
            var multiGet    = new MultiGetDescriptor();

            if (GetParentIdFunc == null)
            {
                itemsToFind.ForEach(id => multiGet.Get <T>(f => f.Id(id).Index(GetIndexById(id))));

                var multiGetResults = await Context.ElasticClient.MultiGetAsync(multiGet).AnyContext();

                foreach (var doc in multiGetResults.Documents)
                {
                    if (!doc.Found)
                    {
                        continue;
                    }

                    results.Documents.Add(doc.Source as T);
                    itemsToFind.Remove(doc.Id);
                }
            }

            // fallback to doing a find
            if (itemsToFind.Count > 0 && (GetParentIdFunc != null || GetDocumentIndexFunc != null))
            {
                results.Documents.AddRange((await FindAsync(new ElasticQuery().WithIds(itemsToFind)).AnyContext()).Documents);
            }

            if (IsCacheEnabled && useCache)
            {
                foreach (var item in results.Documents)
                {
                    await Cache.SetAsync(((IIdentity)item).Id, item, expiresIn.HasValue?DateTime.UtcNow.Add(expiresIn.Value) : DateTime.UtcNow.AddSeconds(RepositoryConstants.DEFAULT_CACHE_EXPIRATION_SECONDS)).AnyContext();
                }
            }

            results.Total = results.Documents.Count;
            return(results);
        }
        public async Task <FindResults <T> > GetByIdsAsync(ICollection <string> ids, PagingOptions paging = null, bool useCache = false, TimeSpan?expiresIn = null)
        {
            if (ids == null || ids.Count == 0)
            {
                return(new FindResults <T>());
            }

            ids = ids.Where(id => !String.IsNullOrEmpty(id)).Distinct().ToList();

            var results = new List <T>();

            if (EnableCache && useCache)
            {
                foreach (var id in ids)
                {
                    var cacheHit = await Cache.GetAsync <T>(GetScopedCacheKey(id)).AnyContext();

                    if (cacheHit.HasValue)
                    {
                        results.Add(cacheHit.Value);
                    }
                }

                var notCachedIds = ids.Except(results.Select(i => i.Id)).ToArray();
                if (notCachedIds.Length == 0)
                {
                    return new FindResults <T> {
                               Documents = results, Total = results.Count
                    }
                }
                ;
            }

            // try using the object id to figure out what index the entity is located in
            var foundItems  = new List <T>();
            var itemsToFind = new List <string>();
            var multiGet    = new MultiGetDescriptor();

            // TODO Use the index..
            foreach (var id in ids.Except(results.Select(i => i.Id)))
            {
                string index = GetIndexName(id);
                if (index != null)
                {
                    multiGet.Get <T>(f => f.Id(id).Index(index).Source(s => s.Exclude("idx")));
                }
                else
                {
                    itemsToFind.Add(id);
                }
            }

#if DEBUG
            _elasticClient.EnableTrace();
            var sw = Stopwatch.StartNew();
#endif
            var multiGetResults = await _elasticClient.MultiGetAsync(multiGet).AnyContext();

#if DEBUG
            sw.Stop();
            _elasticClient.DisableTrace();
            Logger.Trace().Message($"FindAsync: {sw.ElapsedMilliseconds}ms, Serialization Took {multiGetResults.ConnectionStatus.Metrics.SerializationTime}ms, Deserialization Took {multiGetResults.ConnectionStatus.Metrics.DeserializationTime}ms").Write();
#endif

            foreach (var doc in multiGetResults.Documents)
            {
                if (doc.Found)
                {
                    foundItems.Add(doc.Source as T);
                }
                else
                {
                    itemsToFind.Add(doc.Id);
                }
            }

            // fallback to doing a find
            if (itemsToFind.Count > 0)
            {
                foundItems.AddRange((await FindAsync(new ElasticSearchOptions <T>().WithIds(itemsToFind)).AnyContext()).Documents);
            }

            if (EnableCache && useCache && foundItems.Count > 0)
            {
                foreach (var item in foundItems)
                {
                    var expiresAtUtc = expiresIn.HasValue ? DateTime.UtcNow.Add(expiresIn.Value) : DateTime.UtcNow.AddSeconds(RepositoryConstants.DEFAULT_CACHE_EXPIRATION_SECONDS);
                    await Cache.SetAsync(GetScopedCacheKey(item.Id), item, expiresAtUtc).AnyContext();
                }
            }

            results.AddRange(foundItems);
            return(new FindResults <T> {
                Documents = results,
                Total = results.Count
            });
        }
        public virtual async Task <IReadOnlyCollection <T> > GetByIdsAsync(Ids ids, ICommandOptions options = null)
        {
            var idList = ids?.Distinct().Where(i => !String.IsNullOrEmpty(i)).ToList();

            if (idList == null || idList.Count == 0)
            {
                return(EmptyList);
            }

            if (!HasIdentity)
            {
                throw new NotSupportedException("Model type must implement IIdentity.");
            }

            options = ConfigureOptions(options.As <T>());
            if (IsCacheEnabled && options.HasCacheKey())
            {
                throw new ArgumentException("Cache key can't be set when calling GetByIds");
            }

            var hits = new List <FindHit <T> >();

            if (IsCacheEnabled && options.ShouldReadCache())
            {
                hits.AddRange(await GetCachedFindHit(idList).AnyContext());
            }

            var itemsToFind = idList.Except(hits.Select(i => (Id)i.Id)).ToList();

            if (itemsToFind.Count == 0)
            {
                return(hits.Where(h => h.Document != null && ShouldReturnDocument(h.Document, options)).Select(h => h.Document).ToList().AsReadOnly());
            }

            var multiGet = new MultiGetDescriptor();

            foreach (var id in itemsToFind.Where(i => i.Routing != null || !HasParent))
            {
                multiGet.Get <T>(f => {
                    f.Id(id.Value).Index(ElasticIndex.GetIndex(id));
                    if (id.Routing != null)
                    {
                        f.Routing(id.Routing);
                    }

                    return(f);
                });
            }

            var multiGetResults = await _client.MultiGetAsync(multiGet).AnyContext();

            _logger.LogRequest(multiGetResults, options.GetQueryLogLevel());

            foreach (var doc in multiGetResults.Hits)
            {
                hits.Add(((IMultiGetHit <T>)doc).ToFindHit());
                itemsToFind.Remove(new Id(doc.Id, doc.Routing));
            }

            // fallback to doing a find
            if (itemsToFind.Count > 0 && (HasParent || ElasticIndex.HasMultipleIndexes))
            {
                var response = await FindAsync(q => q.Id(itemsToFind.Select(id => id.Value)), o => o.PageLimit(1000)).AnyContext();

                do
                {
                    if (response.Hits.Count > 0)
                    {
                        hits.AddRange(response.Hits.Where(h => h.Document != null));
                    }
                } while (await response.NextPageAsync().AnyContext());
            }

            if (IsCacheEnabled && options.ShouldUseCache())
            {
                await AddDocumentsToCacheAsync(hits, options).AnyContext();
            }

            return(hits.Where(h => h.Document != null && ShouldReturnDocument(h.Document, options)).Select(h => h.Document).ToList().AsReadOnly());
        }