public ElasticsearchPipeline(
ITargetBlock <byte[]> finalTarget,
ElasticsearchPipelineOptions options)
{
var target = CreateBulkRequestBodyBlock(options.TargetBulkSizeInBytes, options.OmitTypeHeaders);
target.LinkTo(finalTarget, new DataflowLinkOptions {
PropagateCompletion = true
});
_target = target;
_completion = finalTarget.Completion;
}
private static ITargetBlock <byte[]> CreateBulkWriterBlock(string url, ElasticsearchPipelineOptions options)
{
var logger = options.Logger;
var postUrl = new Uri(new Uri(url), "_bulk");
var client = options.HttpClientFactory?.Invoke() ?? new HttpClient();
var block = new ActionBlock <byte[]>(async body =>
{
var request = new HttpRequestMessage(HttpMethod.Post, postUrl)
{
Content = new ByteArrayContent(body)
};
request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/x-ndjson");
if (logger?.IsEnabled(LogLevel.Debug) ?? false)
{
logger.LogDebug($"Signing {request.Method} {request.RequestUri} ...");
}
request = await ElasticsearchSigner.SignAsync(request);
if (logger?.IsEnabled(LogLevel.Debug) ?? false)
{
logger.LogDebug($"Sending {request.Method} {request.RequestUri} ...");
}
using (var response = await client.SendAsync(request))
{
if (!response.IsSuccessStatusCode)
{
logger?.LogError($"Got {response.StatusCode} from {postUrl}. Dumping response...");
logger?.LogError(await response.GetDumpAsync());
throw new Exception($"Got {response.StatusCode} response from {postUrl}.");
}
var mediaType = response.Content?.Headers.ContentType?.MediaType?.ToLowerInvariant();
if (mediaType != "application/json")
{
logger?.LogError($"Got {response.StatusCode} with Content-Type: {mediaType} from {postUrl}. Dumping response...");
logger?.LogError(await response.GetDumpAsync());
throw new Exception($"Got {response.StatusCode} but it wasn't JSON.");
}
var responseBody = await response.Content.ReadAsStringAsync();
var bulkResponse = JsonConvert.DeserializeObject <BulkResponse>(responseBody);
if (bulkResponse.Errors)
{
logger?.LogError($"Bulk response from {postUrl} had errors:\n" + responseBody);
throw new Exception($"Bulk operation {postUrl} failed with errors.");
}
logger?.LogInformation($"Wrote {body.Length:n0} bytes /_bulk API");
}
});
block.Completion.ContinueWith(t => client.Dispose());
return(block);
}
private static ITargetBlock <byte[]> CreateBroadcastBulkWriterBlock(string[] urls, ElasticsearchPipelineOptions options)
{
var target = new BroadcastBlock <byte[]>(b => b, new DataflowBlockOptions {
BoundedCapacity = 1
});
var tasks = new List <Task>();
foreach (var url in urls)
{
var subTarget = CreateBulkWriterBlock(url, options);
target.LinkTo(subTarget, new DataflowLinkOptions {
PropagateCompletion = true
});
tasks.Add(subTarget.Completion);
}
return(new TargetBlockWithCompletion <byte[]>(target, Task.WhenAll(tasks)));
}
public ElasticsearchPipeline(
string[] targetUrls,
ElasticsearchPipelineOptions options)
: this(CreateBroadcastBulkWriterBlock(targetUrls, options), options)
{
}