/// <summary>
/// Emit a batch of log events, running to completion synchronously.
/// </summary>
/// <param name="events">The events to emit.</param>
/// <remarks>
/// Override either <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatch(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" /> or <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatchAsync(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" />,
/// not both.
/// </remarks>
protected override void EmitBatch(IEnumerable <LogEvent> events)
{
var logEvents = events.Select(e => new Data.LogEvent(e, e.RenderMessage(_formatProvider)));
if (!logEvents.Any())
{
return;
}
var indexName = string.Format(_indexFormat, DateTime.UtcNow);
var payload = new List <object>();
foreach (var logEvent in logEvents)
{
var document = new Dictionary <string, object>();
document.Add("@timestamp", logEvent.Timestamp);
document.Add("messageTemplate", logEvent.MessageTemplate);
document.Add("level", Enum.GetName(typeof(LogEventLevel), logEvent.Level));
if (logEvent.Exception != null)
{
document.Add("exception", logEvent.Exception);
}
document.Add("message", logEvent.RenderedMessage);
document.Add("fields", logEvent.Properties);
payload.Add(new { index = new { _index = indexName, _type = "logevent" } });
payload.Add(document);
}
_client.Bulk(payload);
}
/**
* ==== Multiple documents with `Bulk`
*
* If you require more control over indexing many documents, you can use the `Bulk` and `BulkAsync` methods and use the descriptors to
* customise the bulk calls.
*
* As with the `IndexMany` methods, documents are sent using the bulk API in a single HTTP request.
* This does mean that consideration should be given to the overall size of the HTTP request. For indexing a large number
* of documents, it may be sensible to perform multiple separate `Bulk` calls, or use <<bulkall-observable, `BulkAllObservable`>>,
* which takes care of a lot of the complexity.
*/
public async Task BulkIndexDocuments()
{
//hide
var people = new [] { new Person {
Id = 1, FirstName = "Martijn", LastName = "Laarman"
} };
var bulkIndexResponse = _client.Bulk(b => b
.Index("people")
.IndexMany(people)
); //<1> synchronous method that returns an IBulkResponse, the same as IndexMany and can be inspected in the same way for errors
// Alternatively, documents can be indexed asynchronously similar to IndexManyAsync
var asyncBulkIndexResponse = await _client.BulkAsync(b => b
.Index("people")
.IndexMany(people)
); //<2> asynchronous method that returns a Task<IBulkResponse> that can be awaited
}
public void CanDeserialize()
{
var json = @"{
""took"": 61,
""errors"": false,
""items"": [{
""update"": {
""_index"": ""test"",
""_type"": ""_doc"",
""_id"": ""1"",
""_version"": 2,
""result"": ""updated"",
""_shards"": {
""total"": 2,
""successful"": 1,
""failed"": 0
},
""_seq_no"": 3,
""_primary_term"": 1,
""get"": {
""_seq_no"": 3,
""_primary_term"": 1,
""found"": true,
""_source"": {
""field1"": ""value1"",
""field2"": ""value2""
}
},
""status"": 200
}
}]
}" ;
var pool = new SingleNodePool(new Uri("http://localhost:9200"));
var connection = new InMemoryConnection(Encoding.UTF8.GetBytes(json));
var settings = new ElasticsearchClientSettings(pool, connection);
var client = new ElasticsearchClient(settings);
var bulkResponse = client.Bulk(new BulkRequest());
bulkResponse.ShouldBeValid();
var item1 = bulkResponse.Items[0].Get;
item1.SeqNo.Should().Be(3);
item1.PrimaryTerm.Should().Be(1);
item1.Found.Should().Be(true);
var simpleObject = bulkResponse.Items[0].Get.Source.As <SimpleObject>();
simpleObject.Field1.Should().Be("value1");
simpleObject.Field2.Should().Be("value2");
}
public bool Commit()
{
try
{
// TODO support controlling bulk sizes
foreach (var deleteByQueryCommand in deleteByQueryCommands)
{
var rsp = elasticsearchClient.DeleteByQuery(deleteByQueryCommand.IndexName, deleteByQueryCommand.TypeName, deleteByQueryCommand.Command);
if (rsp.Success)
{
continue;
}
var sb = new StringBuilder("Error while replicating deletion commands to Elasticsearch on " + rsp.RequestUrl);
sb.Append(string.Format(" (HTTP status code {0}; response: {1})", rsp.HttpStatusCode, rsp.Response));
throw new Exception(sb.ToString(), rsp.OriginalException);
}
deleteByQueryCommands.Clear();
var response = elasticsearchClient.Bulk(bulkCommands);
if (!response.Success)
{
var sb = new StringBuilder("Error while replicating to Elasticsearch on " + response.RequestUrl);
sb.Append(string.Format(" (HTTP status code {0}; response: {1})", response.HttpStatusCode, response.Response));
throw new Exception(sb.ToString(), response.OriginalException);
}
bulkCommands.Clear();
}
catch (Exception e)
{
log.WarnException(
"Failure to replicate changes to Elasticsearch for: " + cfg.Name + ", will continue trying.", e);
replicationStatistics.RecordWriteError(e, database);
hadErrors = true;
return(false);
}
return(true);
}
/// <summary>
/// Emit a batch of log events, running to completion synchronously.
/// </summary>
/// <param name="events">The events to emit.</param>
/// <remarks>
/// Override either <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatch(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" />
/// or <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatchAsync(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" />,
/// not both.
/// </remarks>
protected override void EmitBatch(IEnumerable <LogEvent> events)
{
// ReSharper disable PossibleMultipleEnumeration
if (!events.Any())
{
return;
}
var payload = new List <string>();
foreach (var e in events)
{
var indexName = string.Format(_indexFormat, e.Timestamp);
var action = new { index = new { _index = indexName, _type = _typeName } };
var actionJson = _client.Serializer.Serialize(action, SerializationFormatting.None);
payload.Add(Encoding.UTF8.GetString(actionJson));
var sw = new StringWriter();
_formatter.Format(e, sw);
payload.Add(sw.ToString());
}
_client.Bulk(payload);
}
/// <summary>
/// Shortcut into the Bulk call that indexes the specified objects
/// <para> </para>
/// https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
/// </summary>
/// <param name="client"></param>
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
/// <param name="objects">List of objects to index, Id will be inferred (Id property or IdProperty attribute on type)</param>
/// <param name="index">Override the inferred indexname for T</param>
/// <param name="type">Override the inferred typename for T</param>
public static BulkResponse IndexMany<T>(this ElasticsearchClient client, IEnumerable<T> @objects, IndexName index = null)
where T : class
{
var bulkRequest = CreateIndexBulkRequest(objects, index);
return client.Bulk(bulkRequest);
}