/// <summary> /// Emit a batch of log events, running to completion synchronously. /// </summary> /// <param name="events">The events to emit.</param> /// <remarks> /// Override either <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatch(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" /> or <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatchAsync(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" />, /// not both. /// </remarks> protected override void EmitBatch(IEnumerable <LogEvent> events) { var logEvents = events.Select(e => new Data.LogEvent(e, e.RenderMessage(_formatProvider))); if (!logEvents.Any()) { return; } var indexName = string.Format(_indexFormat, DateTime.UtcNow); var payload = new List <object>(); foreach (var logEvent in logEvents) { var document = new Dictionary <string, object>(); document.Add("@timestamp", logEvent.Timestamp); document.Add("messageTemplate", logEvent.MessageTemplate); document.Add("level", Enum.GetName(typeof(LogEventLevel), logEvent.Level)); if (logEvent.Exception != null) { document.Add("exception", logEvent.Exception); } document.Add("message", logEvent.RenderedMessage); document.Add("fields", logEvent.Properties); payload.Add(new { index = new { _index = indexName, _type = "logevent" } }); payload.Add(document); } _client.Bulk(payload); }
/** * ==== Multiple documents with `Bulk` * * If you require more control over indexing many documents, you can use the `Bulk` and `BulkAsync` methods and use the descriptors to * customise the bulk calls. * * As with the `IndexMany` methods, documents are sent using the bulk API in a single HTTP request. * This does mean that consideration should be given to the overall size of the HTTP request. For indexing a large number * of documents, it may be sensible to perform multiple separate `Bulk` calls, or use <<bulkall-observable, `BulkAllObservable`>>, * which takes care of a lot of the complexity. */ public async Task BulkIndexDocuments() { //hide var people = new [] { new Person { Id = 1, FirstName = "Martijn", LastName = "Laarman" } }; var bulkIndexResponse = _client.Bulk(b => b .Index("people") .IndexMany(people) ); //<1> synchronous method that returns an IBulkResponse, the same as IndexMany and can be inspected in the same way for errors // Alternatively, documents can be indexed asynchronously similar to IndexManyAsync var asyncBulkIndexResponse = await _client.BulkAsync(b => b .Index("people") .IndexMany(people) ); //<2> asynchronous method that returns a Task<IBulkResponse> that can be awaited }
public void CanDeserialize() { var json = @"{ ""took"": 61, ""errors"": false, ""items"": [{ ""update"": { ""_index"": ""test"", ""_type"": ""_doc"", ""_id"": ""1"", ""_version"": 2, ""result"": ""updated"", ""_shards"": { ""total"": 2, ""successful"": 1, ""failed"": 0 }, ""_seq_no"": 3, ""_primary_term"": 1, ""get"": { ""_seq_no"": 3, ""_primary_term"": 1, ""found"": true, ""_source"": { ""field1"": ""value1"", ""field2"": ""value2"" } }, ""status"": 200 } }] }" ; var pool = new SingleNodePool(new Uri("http://localhost:9200")); var connection = new InMemoryConnection(Encoding.UTF8.GetBytes(json)); var settings = new ElasticsearchClientSettings(pool, connection); var client = new ElasticsearchClient(settings); var bulkResponse = client.Bulk(new BulkRequest()); bulkResponse.ShouldBeValid(); var item1 = bulkResponse.Items[0].Get; item1.SeqNo.Should().Be(3); item1.PrimaryTerm.Should().Be(1); item1.Found.Should().Be(true); var simpleObject = bulkResponse.Items[0].Get.Source.As <SimpleObject>(); simpleObject.Field1.Should().Be("value1"); simpleObject.Field2.Should().Be("value2"); }
public bool Commit() { try { // TODO support controlling bulk sizes foreach (var deleteByQueryCommand in deleteByQueryCommands) { var rsp = elasticsearchClient.DeleteByQuery(deleteByQueryCommand.IndexName, deleteByQueryCommand.TypeName, deleteByQueryCommand.Command); if (rsp.Success) { continue; } var sb = new StringBuilder("Error while replicating deletion commands to Elasticsearch on " + rsp.RequestUrl); sb.Append(string.Format(" (HTTP status code {0}; response: {1})", rsp.HttpStatusCode, rsp.Response)); throw new Exception(sb.ToString(), rsp.OriginalException); } deleteByQueryCommands.Clear(); var response = elasticsearchClient.Bulk(bulkCommands); if (!response.Success) { var sb = new StringBuilder("Error while replicating to Elasticsearch on " + response.RequestUrl); sb.Append(string.Format(" (HTTP status code {0}; response: {1})", response.HttpStatusCode, response.Response)); throw new Exception(sb.ToString(), response.OriginalException); } bulkCommands.Clear(); } catch (Exception e) { log.WarnException( "Failure to replicate changes to Elasticsearch for: " + cfg.Name + ", will continue trying.", e); replicationStatistics.RecordWriteError(e, database); hadErrors = true; return(false); } return(true); }
/// <summary> /// Emit a batch of log events, running to completion synchronously. /// </summary> /// <param name="events">The events to emit.</param> /// <remarks> /// Override either <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatch(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" /> /// or <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatchAsync(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" />, /// not both. /// </remarks> protected override void EmitBatch(IEnumerable <LogEvent> events) { // ReSharper disable PossibleMultipleEnumeration if (!events.Any()) { return; } var payload = new List <string>(); foreach (var e in events) { var indexName = string.Format(_indexFormat, e.Timestamp); var action = new { index = new { _index = indexName, _type = _typeName } }; var actionJson = _client.Serializer.Serialize(action, SerializationFormatting.None); payload.Add(Encoding.UTF8.GetString(actionJson)); var sw = new StringWriter(); _formatter.Format(e, sw); payload.Add(sw.ToString()); } _client.Bulk(payload); }
/// <summary> /// Shortcut into the Bulk call that indexes the specified objects /// <para> </para> /// https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html /// </summary> /// <param name="client"></param> /// <typeparam name="T">The type used to infer the default index and typename</typeparam> /// <param name="objects">List of objects to index, Id will be inferred (Id property or IdProperty attribute on type)</param> /// <param name="index">Override the inferred indexname for T</param> /// <param name="type">Override the inferred typename for T</param> public static BulkResponse IndexMany<T>(this ElasticsearchClient client, IEnumerable<T> @objects, IndexName index = null) where T : class { var bulkRequest = CreateIndexBulkRequest(objects, index); return client.Bulk(bulkRequest); }