/// <summary>
/// Shortcut into the Bulk call that indexes the specified objects
/// <para> </para>
/// https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
/// </summary>
/// <param name="client"></param>
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
/// <param name="objects">List of objects to index, Id will be inferred (Id property or IdProperty attribute on type)</param>
/// <param name="index">Override the inferred indexname for T</param>
/// <param name="type">Override the inferred typename for T</param>
public static Task<BulkResponse> IndexManyAsync<T>(this ElasticsearchClient client, IEnumerable<T> objects, IndexName index = null,
CancellationToken cancellationToken = default
)
where T : class
{
var bulkRequest = CreateIndexBulkRequest(objects, index);
return client.BulkAsync(bulkRequest, cancellationToken);
}
/// <summary>
/// Connect to ElasticSearch server, before connected all log entries will be catched in queue
/// </summary>
/// <param name="url"></param>
public static void ConnectElasticSearch(string url = "http://localhost:9200/")
{
var node = new Uri(url);
var config = new ConnectionConfiguration(node)
.EnableTrace(false)
.EnableMetrics(false)
.UsePrettyRequests(false)
.UsePrettyResponses(false);
client = new ElasticsearchClient(config);
consoleOnly = false;
CancellationToken ct = stopToken.Token;
Task taskConsumeQueue = Task.Run(() =>
{
while (true)
{
// Bulk insert cached logs every 1 second
while (logEntries.IsEmpty)
{
System.Threading.Thread.Sleep(1000);
// <- Stop()
if (ct.IsCancellationRequested)
{
return;
}
}
int batchLength = 0;
var builder = new StringBuilder();
var timestamp = "logger-" + DateTime.UtcNow.ToString("yyyy-MM-dd");
var indexOp = "{ \"index\" : { \"_index\" : \"" + timestamp + "\", \"_type\" : \"log\" } }";
while (logEntries.IsEmpty == false && batchLength < 100)
{
JObject entry;
if (logEntries.TryDequeue(out entry))
{
builder.AppendLine(indexOp);
builder.AppendLine(entry.ToString(Formatting.None));
batchLength++;
}
}
if (batchLength > 0)
{
client.BulkAsync(builder.ToString());
}
}
});
}
/**
* ==== Multiple documents with `Bulk`
*
* If you require more control over indexing many documents, you can use the `Bulk` and `BulkAsync` methods and use the descriptors to
* customise the bulk calls.
*
* As with the `IndexMany` methods, documents are sent using the bulk API in a single HTTP request.
* This does mean that consideration should be given to the overall size of the HTTP request. For indexing a large number
* of documents, it may be sensible to perform multiple separate `Bulk` calls, or use <<bulkall-observable, `BulkAllObservable`>>,
* which takes care of a lot of the complexity.
*/
public async Task BulkIndexDocuments()
{
//hide
var people = new [] { new Person {
Id = 1, FirstName = "Martijn", LastName = "Laarman"
} };
var bulkIndexResponse = _client.Bulk(b => b
.Index("people")
.IndexMany(people)
); //<1> synchronous method that returns an IBulkResponse, the same as IndexMany and can be inspected in the same way for errors
// Alternatively, documents can be indexed asynchronously similar to IndexManyAsync
var asyncBulkIndexResponse = await _client.BulkAsync(b => b
.Index("people")
.IndexMany(people)
); //<2> asynchronous method that returns a Task<IBulkResponse> that can be awaited
}