private void RefreshOnCompleted()
{
if (!_partitionedBulkRequest.RefreshOnCompleted)
{
return;
}
var indices = _partitionedBulkRequest.RefreshIndices ?? _partitionedBulkRequest.Index;
if (indices == null)
{
return;
}
var rc = _partitionedBulkRequest switch
{
IHelperCallable helperCallable when helperCallable.ParentMetaData is not null => helperCallable.ParentMetaData,
_ => RequestMetaDataFactory.BulkHelperRequestMetaData(),
};
var request = new IndexManagement.RefreshRequest(indices);
if (rc is not null)
{
request.RequestConfiguration = new RequestConfiguration {
RequestMetaData = rc
}
}
;
var refresh = _client.Indices.Refresh(request);
if (!refresh.IsValid)
{
throw Throw($"Refreshing after all documents have indexed failed", refresh.ApiCall);
}
}
private async Task <BulkAllResponse> BulkAsync(IList <T> buffer, long page, int backOffRetries)
{
_compositeCancelToken.ThrowIfCancellationRequested();
var request = _partitionedBulkRequest;
var response = await _client.BulkAsync(s =>
{
s.Index(request.Index);
s.Timeout(request.Timeout);
if (request.BufferToBulk is not null)
{
request.BufferToBulk(s, buffer);
}
else
{
s.IndexMany(buffer);
}
if (!string.IsNullOrEmpty(request.Pipeline))
{
s.Pipeline(request.Pipeline);
}
if (request.Routing != null)
{
s.Routing(request.Routing);
}
if (request.WaitForActiveShards.HasValue)
{
s.WaitForActiveShards(request.WaitForActiveShards.ToString());
}
switch (_partitionedBulkRequest)
{
case IHelperCallable helperCallable when helperCallable.ParentMetaData is not null:
s.RequestConfiguration(rc => rc.RequestMetaData(helperCallable.ParentMetaData));
break;
default:
s.RequestConfiguration(rc => rc.RequestMetaData(RequestMetaDataFactory.BulkHelperRequestMetaData()));
break;
}
}, _compositeCancelToken).ConfigureAwait(false);
_compositeCancelToken.ThrowIfCancellationRequested();
_bulkResponseCallback?.Invoke(response);
if (!response.ApiCall.Success)
{
return(await HandleBulkRequestAsync(buffer, page, backOffRetries, response).ConfigureAwait(false));
}
var retryableDocuments = new List <T>();
var droppedDocuments = new List <Tuple <BulkResponseItemBase, T> >();
foreach (var documentWithResponse in response.Items.Zip(buffer, Tuple.Create))
{
if (documentWithResponse.Item1.IsValid)
{
continue;
}
if (_retryPredicate(documentWithResponse.Item1, documentWithResponse.Item2))
{
retryableDocuments.Add(documentWithResponse.Item2);
}
else
{
droppedDocuments.Add(documentWithResponse);
}
}
HandleDroppedDocuments(droppedDocuments, response);
if (retryableDocuments.Count > 0 && backOffRetries < _backOffRetries)
{
return(await RetryDocumentsAsync(page, ++backOffRetries, retryableDocuments).ConfigureAwait(false));
}
if (retryableDocuments.Count > 0)
{
throw ThrowOnBadBulk(response, $"Bulk indexing failed and after retrying {backOffRetries} times");
}
request.BackPressure?.Release();
return(new BulkAllResponse {
Retries = backOffRetries, Page = page, Items = response.Items
});
}