/// <summary> /// Shortcut into the Bulk call that indexes the specified objects /// <para> </para> /// https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html /// </summary> /// <param name="client"></param> /// <typeparam name="T">The type used to infer the default index and typename</typeparam> /// <param name="objects">List of objects to index, Id will be inferred (Id property or IdProperty attribute on type)</param> /// <param name="index">Override the inferred indexname for T</param> /// <param name="type">Override the inferred typename for T</param> public static Task<BulkResponse> IndexManyAsync<T>(this ElasticsearchClient client, IEnumerable<T> objects, IndexName index = null, CancellationToken cancellationToken = default ) where T : class { var bulkRequest = CreateIndexBulkRequest(objects, index); return client.BulkAsync(bulkRequest, cancellationToken); }
/// <summary> /// Connect to ElasticSearch server, before connected all log entries will be catched in queue /// </summary> /// <param name="url"></param> public static void ConnectElasticSearch(string url = "http://localhost:9200/") { var node = new Uri(url); var config = new ConnectionConfiguration(node) .EnableTrace(false) .EnableMetrics(false) .UsePrettyRequests(false) .UsePrettyResponses(false); client = new ElasticsearchClient(config); consoleOnly = false; CancellationToken ct = stopToken.Token; Task taskConsumeQueue = Task.Run(() => { while (true) { // Bulk insert cached logs every 1 second while (logEntries.IsEmpty) { System.Threading.Thread.Sleep(1000); // <- Stop() if (ct.IsCancellationRequested) { return; } } int batchLength = 0; var builder = new StringBuilder(); var timestamp = "logger-" + DateTime.UtcNow.ToString("yyyy-MM-dd"); var indexOp = "{ \"index\" : { \"_index\" : \"" + timestamp + "\", \"_type\" : \"log\" } }"; while (logEntries.IsEmpty == false && batchLength < 100) { JObject entry; if (logEntries.TryDequeue(out entry)) { builder.AppendLine(indexOp); builder.AppendLine(entry.ToString(Formatting.None)); batchLength++; } } if (batchLength > 0) { client.BulkAsync(builder.ToString()); } } }); }
/** * ==== Multiple documents with `Bulk` * * If you require more control over indexing many documents, you can use the `Bulk` and `BulkAsync` methods and use the descriptors to * customise the bulk calls. * * As with the `IndexMany` methods, documents are sent using the bulk API in a single HTTP request. * This does mean that consideration should be given to the overall size of the HTTP request. For indexing a large number * of documents, it may be sensible to perform multiple separate `Bulk` calls, or use <<bulkall-observable, `BulkAllObservable`>>, * which takes care of a lot of the complexity. */ public async Task BulkIndexDocuments() { //hide var people = new [] { new Person { Id = 1, FirstName = "Martijn", LastName = "Laarman" } }; var bulkIndexResponse = _client.Bulk(b => b .Index("people") .IndexMany(people) ); //<1> synchronous method that returns an IBulkResponse, the same as IndexMany and can be inspected in the same way for errors // Alternatively, documents can be indexed asynchronously similar to IndexManyAsync var asyncBulkIndexResponse = await _client.BulkAsync(b => b .Index("people") .IndexMany(people) ); //<2> asynchronous method that returns a Task<IBulkResponse> that can be awaited }