public bool BulkIndexDesc <T>(IEnumerable <T> inputObject, bool isAsync = false) where T : class { try { var descriptor = new BulkDescriptor(); foreach (T pageview in inputObject) { T item = pageview; descriptor.Index <T>(op => op.Document(item)); } if (isAsync) { _client.BulkAsync(descriptor); } else { _client.Bulk(descriptor); } return(true); } catch (Exception ex) { throw; } }
public async Task <Response> AddEntitiesAsync <TEntity>(IList <TEntity> entities) where TEntity : class { var response = new Response(); if (entities == null) { return(response); } if (!entities.Any()) { return(response); } var indexName = ElasticSearchExtensions.GetIndexNameFrom <TEntity>(); var indexExistsResponse = await _elasticClient.Indices.ExistsAsync(indexName) .ConfigureAwait(false); if (!indexExistsResponse.Exists) { var indexCreateResponse = await CreateIndexAsync <TEntity>(indexName) .ConfigureAwait(false); if (!indexCreateResponse.IsValid) { response.IsOk = false; response.ErrorMessage = indexCreateResponse.OriginalException.Message; response.Exception = indexCreateResponse.OriginalException; return(response); } } var bulkInsert = await _elasticClient.BulkAsync(b => b .IndexMany(entities, (op, item) => op.Index(indexName) .Id(GetEntityId(item))) ) .ConfigureAwait(false); await _elasticClient.Indices.RefreshAsync(indexName) .ConfigureAwait(false); if (!bulkInsert.Errors) { return(response); } response.IsOk = false; response.ErrorMessage = bulkInsert.OriginalException.Message; response.Exception = bulkInsert.OriginalException; return(response); }
public async Task BulkIndex(IEnumerable <T> documents, Func <TypeMappingDescriptor <T>, ITypeMapping> mapper = null, bool switchIndex = true) { _indexRotator.InitAlias <T>(_aliasName, m => mapper != null ? mapper(m) : m.AutoMap()); var newIndexName = _indexRotator.GetSecondaryIndex(_aliasName, mapper); var bulk = documents.Aggregate(new BulkDescriptor(newIndexName), (acc, x) => acc.Index <T>(p => p.Index(newIndexName).Document(x))); await _esClient.BulkAsync(bulk); if (switchIndex) { await _indexRotator.SwitchIndex(_aliasName); } }
protected override async Task ProduceMessages( StreamName stream, IEnumerable <ProducedMessage> messages, ElasticProduceOptions?options, CancellationToken cancellationToken = default ) { var documents = messages.Select(x => x.Message); var mode = options?.ProduceMode ?? ProduceMode.Create; var bulk = GetOp(new BulkDescriptor(stream.ToString())); var result = await _elasticClient.BulkAsync(bulk, cancellationToken); if (!result.IsValid) { if (result.DebugInformation.Contains("version conflict")) { SubscriptionsEventSource.Log.Warn("ElasticProducer: version conflict"); } else { throw result.OriginalException ?? throw new InvalidOperationException(result.DebugInformation); } } BulkDescriptor GetOp(BulkDescriptor descriptor) => mode switch { ProduceMode.Create => descriptor.CreateMany(documents), ProduceMode.Index => descriptor.IndexMany(documents), _ => throw new ArgumentOutOfRangeException() }; } }
public async Task ReIndexBulkAsync() { await elasticClient.DeleteByQueryAsync <ProductViewModel>(q => q.Index("products").MatchAll()); var products = productRepository.GetAll().ToArray(); await elasticClient.BulkAsync(b => b.Index("products").IndexMany(products)); }
private Task <BulkResponse> BulkData(IBulkRequest request) { var response = _client.BulkAsync(request); if (!response.Result.IsValid) { if ((response.Result.OriginalException != null && response.Result.OriginalException.Message.Contains("The underlying connection was closed: A connection that was expected to be kept alive was closed by the server")) || (response.Result.ItemsWithErrors != null && response.Result.ItemsWithErrors.First().Status == 503)) { _logger.Warn("Elasticsearch overload, retrying"); throw new TooManyRequestsException(); } if (response.Exception != null) { foreach (var ex in response.Exception.InnerExceptions) { _logger.Error(ex, ex.Message); } } throw new HttpException(response.Result.ItemsWithErrors.First().Status, "Something failed trying to insert data into the bulk service", response.Exception); } return(response); }
private static IEnumerable <Func <object> > DocumentCommandsAsync(IElasticClient elastic) { return(new List <Func <object> > { () => elastic.BulkAsync( new BulkRequest("test_index") { Operations = new List <IBulkOperation> { new BulkCreateOperation <Post>(new Post { Id = 1, Title = "BulkCreateOperation" }) } }), () => elastic.CreateAsync(new CreateRequest <Post>(new Post { Id = 2, Title = "CreateRequest" }, "test_index")), () => elastic.CreateDocumentAsync(new Post { Id = 3, Title = "CreateDocument" }), () => elastic.CountAsync <Post>(), () => elastic.SearchAsync <Post>(s => s.MatchAll()), () => elastic.DeleteByQueryAsync(new DeleteByQueryRequest("test_index") { Size = 0 }) }); }
/// <summary> /// Shortcut into the Bulk call that deletes the specified objects /// <para> </para>http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html /// </summary> /// <param name="client"></param> /// <typeparam name="T">The type used to infer the default index and typename</typeparam> /// <param name="objects">List of objects to delete</param> /// <param name="index">Override the inferred indexname for T</param> /// <param name="type">Override the inferred typename for T</param> public static Task <IBulkResponse> DeleteManyAsync <T>(this IElasticClient client, IEnumerable <T> objects, string index = null, string type = null, CancellationToken cancellationToken = default(CancellationToken)) where T : class { var bulkRequest = CreateDeleteBulkRequest(objects, index, type); return(client.BulkAsync(bulkRequest, cancellationToken)); }
public static async Task <IBulkResponse> BulkAsync <T>( this IElasticClient client, IEnumerable <BulkItem <T> > items, string routing = null, string indexName = null) where T : class { indexName = indexName ?? client.ConnectionSettings.DefaultIndex; var response = await client.BulkAsync(bulkDescriptor => { bulkDescriptor = bulkDescriptor.Index(indexName); if (routing != null) { bulkDescriptor = bulkDescriptor.Routing(routing); } foreach (var item in items) { switch (item.Operation) { case BulkOperation.Index: bulkDescriptor = bulkDescriptor.Index <T>( indexDescriptor => indexDescriptor .Routing(item.Routing) .Document(item.Document) .Id(item.Id)); break; case BulkOperation.Create: bulkDescriptor = bulkDescriptor.Create <T>( createDescriptor => createDescriptor .Routing(item.Routing) .Document(item.Document) .Id(item.Id)); break; case BulkOperation.Delete: bulkDescriptor = bulkDescriptor.Delete <T>( deleteDescriptor => deleteDescriptor .Routing(item.Routing) .Id(item.Id)); break; case BulkOperation.Upsert: bulkDescriptor = bulkDescriptor.Update <T>( updateDescriptor => updateDescriptor .Routing(item.Routing) .Doc(item.Document) .DocAsUpsert(true) .Id(item.Id)); break; } } return(bulkDescriptor); }); return(response); }
/// <summary> /// Shortcut into the Bulk call that indexes the specified objects /// <para> </para>http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html /// </summary> /// <param name="client"></param> /// <typeparam name="T">The type used to infer the default index and typename</typeparam> /// <param name="objects">List of objects to index, Id will be inferred (Id property or IdProperty attribute on type)</param> /// <param name="index">Override the inferred indexname for T</param> /// <param name="type">Override the inferred typename for T</param> public static Task <IBulkResponse> IndexManyAsync <T>(this IElasticClient client, IEnumerable <T> objects, string index = null, string type = null) where T : class { var bulkRequest = CreateIndexBulkRequest(objects, index, type); return(client.BulkAsync(bulkRequest)); }
public async Task InsertRangeAsync <T>(IEnumerable <T> entity, string index) where T : class { var indexName = index.GetIndex <T>(); var exists = await IndexExistsAsync(indexName); if (!exists) { await((ElasticClient)_elasticClient).CreateIndexAsync <T>(indexName); await AddAliasAsync(indexName, typeof(T).Name); } var bulkRequest = new BulkRequest(indexName) { Operations = new List <IBulkOperation>() }; var operations = entity.Select(o => new BulkIndexOperation <T>(o)).Cast <IBulkOperation>().ToList(); bulkRequest.Operations = operations; var response = await _elasticClient.BulkAsync(bulkRequest); if (!response.IsValid) { throw new Exception("批量新增数据失败:" + response.OriginalException.Message); } }
public async Task <AppendEventsResult> AppendEvents( StreamName stream, ExpectedStreamVersion expectedVersion, IReadOnlyCollection <StreamEvent> events, CancellationToken cancellationToken ) { var streamName = stream.ToString(); var documents = events.Select(AsDocument).ToArray(); var bulk = new BulkDescriptor(_options.IndexName).CreateMany(documents).Refresh(Refresh.WaitFor); var result = await _client.BulkAsync(bulk, cancellationToken); return(result.IsValid ? new AppendEventsResult(0, documents.Last().StreamPosition + 1) : throw new ApplicationException($"Unable to add events: {result.DebugInformation}")); PersistedEvent AsDocument(StreamEvent evt) => new( evt.Id.ToString(), TypeMap.Instance.GetTypeName(evt.Payload !), evt.Position + 1, evt.ContentType, streamName, (ulong)evt.Position + 1, evt.Payload, evt.Metadata.ToHeaders(), DateTime.Now ); }
public virtual Task <IBulkResponse> BulkAsync(IBulkRequest request, string callerName = "") { var timer = Stopwatch.StartNew(); var result = _client.BulkAsync(request); SendLog(null, null, timer.ElapsedMilliseconds, $"Bulk Async {callerName}"); return(result); }
/// <summary> /// Shortcut into the Bulk call that indexes the specified objects /// <para> </para> /// https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html /// </summary> /// <param name="client"></param> /// <typeparam name="T">The type used to infer the default index and typename</typeparam> /// <param name="objects">List of objects to index, Id will be inferred (Id property or IdProperty attribute on type)</param> /// <param name="index">Override the inferred indexname for T</param> /// <param name="type">Override the inferred typename for T</param> public static Task <BulkResponse> IndexManyAsync <T>(this IElasticClient client, IEnumerable <T> objects, IndexName index = null, CancellationToken cancellationToken = default ) where T : class { var bulkRequest = CreateIndexBulkRequest(objects, index); return(client.BulkAsync(bulkRequest, cancellationToken)); }
public async Task <IActionResult> BulkInsert([FromBody] List <Company> companies) { try { var queryResponse = await _elasticClient.BulkAsync(x => x .CreateMany(companies) .Index(IndexName)); if (queryResponse.IsValid) { return(Ok(queryResponse.Items.Count)); } return(BadRequest(queryResponse.ServerError.Error)); } catch (System.Exception ex) { return(BadRequest(ex.Message)); } }
/// <summary> /// 添加到索引 /// </summary> /// <typeparam name="T"></typeparam> /// <param name="client"></param> /// <param name="indexName"></param> /// <param name="data"></param> /// <returns></returns> public static async Task AddToIndexAsync <T>(this IElasticClient client, string indexName, params T[] data) where T : class, IElasticSearchIndex { var bulk = new BulkRequest(indexName) { Operations = ConvertHelper.NotNullList(data).Select(x => new BulkIndexOperation <T>(x)).ToArray() }; var response = await client.BulkAsync(bulk); response.ThrowIfException(); }
/// <summary> /// 批量保存文档 /// </summary> /// <typeparam name="TDocument">文档类型</typeparam> /// <param name="documents">文档对象列表</param> /// <param name="index">索引名称。注意:必须小写</param> /// <param name="timeout">超时时间间隔。单位:毫秒,默认值:300000,即5分钟</param> /// <param name="cancellationToken">取消令牌</param> public async Task <BulkResponse> BulkSaveAsync <TDocument>(IEnumerable <TDocument> documents, string index = null, double timeout = 300000, CancellationToken cancellationToken = default) where TDocument : class { index = GetIndexName(Helper.SafeIndexName <TDocument>(index)); return(await _client.BulkAsync(x => x .Index(index) .IndexMany(documents) .Timeout(timeout), cancellationToken)); }
private async Task BulkAddOrUpdate <T, TKey>(string indexName, List <T> list) where T : class { var bulk = new BulkRequest(indexName) { Operations = new List <IBulkOperation>() }; foreach (var item in list) { bulk.Operations.Add(new BulkIndexOperation <T>(item)); } var response = await _esClient.BulkAsync(bulk); if (response.Errors) { throw new ElasticSearchException( $"Bulk InsertOrUpdate Document failed at index {indexName} :{response.ServerError.Error.Reason}"); } }
public static Task <IBulkResponse> IndexManyAsync <T>(this IElasticClient client, IEnumerable <T> objects, Func <T, string> getParent, Func <T, string> getIndex = null, string type = null) where T : class { if (getParent == null && getIndex == null) { return(client.IndexManyAsync(objects, null, type)); } var indexBulkRequest = CreateIndexBulkRequest(objects, getIndex, type, getParent); return(client.BulkAsync(indexBulkRequest)); }
public async Task PublishToElastic(IClass createdClass) { List <IBulkOperation> bulkOps = new List <IBulkOperation>(); var createReq = new BulkIndexOperation <Class>(createdClass as Class); createReq.Index = ElasticConfig.Indices.Main.Name(); bulkOps.Add(createReq); var createReqLog = new BulkIndexOperation <Activity <string> >(new Activity <string>() { Id = Guid.NewGuid().ToString(), Timestamp = DateTime.UtcNow, Payload = new ActivityPayload <string>() { Data = string.Format($"SearchProvider: Create(), Created Class - {JsonConvert.SerializeObject(createdClass)}") }, ActivityType = ActivityType.ClassCreate }); createReqLog.Index = ElasticConfig.Indices.Logs.Name(); bulkOps.Add(createReqLog); await _elasticProvider.BulkAsync(new BulkRequest() { Operations = bulkOps }); }
public async Task SaveBulkAsync(T[] entities) { var result = await _elasticClient.BulkAsync(b => b.Index("service_management").IndexMany(entities)); if (result.Errors) { // the response can be inspected for errors foreach (var item in result.ItemsWithErrors) { Console.WriteLine("Failed to index document {0} : {1}", item.Id, item.Error); } } }
/// <summary> /// Inserts/ Index documents /// </summary> /// <param name="entities"></param> /// <returns></returns> public async Task BulkInsertAsync(IEnumerable <QuotesModel> entities) { var request = new BulkDescriptor(); foreach (var entity in entities) { request .Index <QuotesModel>(op => op .Id(Guid.NewGuid().ToString()) .Document(entity)); } await Client.BulkAsync(request); }
public async Task SaveBulkAsync(IEnumerable <AuditEntity> audits) { var result = await _elasticClient.BulkAsync(b => b.Index("audits").IndexMany(audits)); if (result.Errors) { // the response can be inspected for errors foreach (var itemWithError in result.ItemsWithErrors) { _logger.LogError("Failed to index document {0}: {1}", itemWithError.Id, itemWithError.Error); } } }
public async Task SaveBulkAsync(Product[] products) { _cache.AddRange(products); var result = await _elasticClient.BulkAsync(b => b.Index("products").IndexMany(products)); if (result.Errors) { // the response can be inspected for errors foreach (var itemWithError in result.ItemsWithErrors) { _logger.LogError("Failed to index document {0}: {1}", itemWithError.Id, itemWithError.Error); } } }
public async Task SaveBulkAsync(Address[] addresses) { _cache.AddRange(addresses); // TODO : WARNING HARDCODED INDEX var result = await _elasticClient.BulkAsync(b => b.Index("addresses").IndexMany(addresses)); if (result.Errors) { // the response can be inspected for errors foreach (var itemWithError in result.ItemsWithErrors) { _logger.LogError("Failed to index document {0}: {1}", itemWithError.Id, itemWithError.Error); } } }
private async Task UpdateData(List <Feature> features, string alias) { var result = await _elasticClient.BulkAsync(bulk => { foreach (var feature in features) { bulk.Index <Feature>(i => i.Index(alias).Document(feature).Id(GetId(feature))); } return(bulk); }); if (result.IsValid == false) { result.ItemsWithErrors.ToList().ForEach(i => _logger.LogError($"Inserting {i.Id} failed with error: {i.Error?.Reason ?? string.Empty} caused by: {i.Error?.CausedBy?.Reason ?? string.Empty}")); } }
/** * ==== Multiple documents with bulk * * If you require finer grained control over indexing many documents you can use the `Bulk` and `BulkAsync` methods and use the descriptors to * customise the bulk calls. * * As with the `IndexMany` methods above, documents are sent to the `_bulk` endpoint in a single HTTP request. * This does mean that consideration will need to be given to the overall size of the HTTP request. For indexing large numbers * of documents it may be sensible to perform multiple separate `Bulk` calls. */ public async Task BulkIndexDocuments() { //hide var people = new [] { new Person { Id = 1, FirstName = "Martijn", LastName = "Laarman" } }; var bulkIndexResponse = client.Bulk(b => b .Index("people") .IndexMany(people)); //<1> synchronous method that returns an IBulkResponse, the same as IndexMany and can be inspected in the same way for errors // Alternatively, documents can be indexed asynchronously similar to IndexManyAsync var asyncBulkIndexResponse = await client.BulkAsync(b => b .Index("people") .IndexMany(people)); //<2> asynchronous method that returns a Task<IBulkResponse> that can be awaited }
/// <summary> /// Inserts documents in bulk into Elastic Search. /// </summary> /// <param name="docs">The documents to insert.</param> public virtual async Task <IBulkResponse> Index(IEnumerable <T> docs) { var request = new BulkDescriptor(); foreach (var req in docs) { var reqES = PrepareDoc(req); request.Index <T>(op => op .Id(reqES.Id) .Index(_serviceConfig.IndexName) .Document(reqES) ); } return(await _client.BulkAsync(request)); }
public override async Task ProfileAsync(IElasticClient client, ColoredConsoleWriter output) { for (var i = 0; i < _iterations; i++) { var bulkResponse = await client.BulkAsync(b => b .IndexMany(Developer.Generator.Generate(_itemsPerIteration), (bd, d) => bd .Index(Infer.Index<Developer>()) .Document(d) )).ConfigureAwait(false); if (!bulkResponse.IsValid) if (bulkResponse.Errors) foreach (var error in bulkResponse.ItemsWithErrors) output.WriteOrange($"error with id {error.Id}. message: {error.Error.Reason}"); } }
public async Task<IBulkResponse> Handle(SaveOrUpdateMovieTrailerCommand request, CancellationToken cancellationToken) { if (!request.MovieTrailers.Any()) { return null; } var descriptor = new BulkDescriptor(); foreach (var movieTrailer in request.MovieTrailers) { descriptor.Index<MovieTrailer>(op => op.Document(movieTrailer)); } var response = await elasticClient.BulkAsync(descriptor); return response; }
public async Task End(Exception ex = null) { if (ex != null) { return; } if (_pendingSize > 0) { // Index the pending docs var response = await _client.BulkAsync(_pendingDocs.Consistency(Elasticsearch.Net.Consistency.Quorum)); if (response.Errors) { throw new StorageException(response); } } }