public async Task Ingest(IEnumerable <Record> entries)
{
var activity = new IngestActivity();
var operations = new List <IBulkOperation>(BatchSize);
foreach (var entry in entries)
{
var indexName = GetIndex(entry);
operations.Add(new BulkIndexOperation <object>(entry.JsonDocument)
{
Id = !string.IsNullOrWhiteSpace(entry.Id) ? new Id(entry.Id) : null,
Index = indexName,
Type = entry.TypeName
});
activity.AddDocument(indexName, entry.IndexName, entry.TypeName, entry.JsonDocument.Length);
if (operations.Count == BatchSize)
{
await IngestBatch(operations, activity);
operations.Clear();
activity = new IngestActivity();
}
}
if (operations.Count > 0)
{
await IngestBatch(operations, activity);
}
}
private async Task IngestBatch(IList <IBulkOperation> indexOperations, IngestActivity activity)
{
var request = new BulkRequest
{
Refresh = false,
Operations = indexOperations
};
var result = await client.BulkAsync(request);
foreach (var item in result.ItemsWithErrors)
{
// TODO
}
activity.CompleteIngestion();
activityLogger.LogActivity(activity);
}
public void LogActivity(IngestActivity activity)
{
// TODO - hmm.
}