public ScrollAllObservable(IOpenSearchClient client, IScrollAllRequest scrollAllRequest, CancellationToken cancellationToken = default)
{
_scrollAllRequest = scrollAllRequest;
_searchRequest = scrollAllRequest?.Search ?? new SearchRequest <T>();
switch (_scrollAllRequest)
{
case IHelperCallable helperCallable when helperCallable.ParentMetaData is object:
_searchRequest.RequestParameters.SetRequestMetaData(helperCallable.ParentMetaData);
break;
default:
_searchRequest.RequestParameters.SetRequestMetaData(RequestMetaDataFactory.ScrollHelperRequestMetaData());
break;
}
if (_searchRequest.Sort == null)
{
_searchRequest.Sort = FieldSort.ByDocumentOrder;
}
_searchRequest.RequestParameters.Scroll = _scrollAllRequest.ScrollTime.ToTimeSpan();
_client = client;
_compositeCancelTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
_compositeCancelToken = _compositeCancelTokenSource.Token;
_backPressure = _scrollAllRequest.BackPressure;
}
private void RefreshOnCompleted()
{
if (!_partitionedBulkRequest.RefreshOnCompleted)
{
return;
}
var indices = _partitionedBulkRequest.RefreshIndices ?? _partitionedBulkRequest.Index;
if (indices == null)
{
return;
}
var refresh = _client.Indices.Refresh(indices, r => r.RequestConfiguration(rc =>
{
switch (_partitionedBulkRequest)
{
case IHelperCallable helperCallable when helperCallable.ParentMetaData is object:
rc.RequestMetaData(helperCallable.ParentMetaData);
break;
default:
rc.RequestMetaData(RequestMetaDataFactory.BulkHelperRequestMetaData());
break;
}
return(rc);
}));
if (!refresh.IsValid)
{
throw Throw($"Refreshing after all documents have indexed failed", refresh.ApiCall);
}
}
public SnapshotObservable(IOpenSearchClient opensearchClient, ISnapshotRequest snapshotRequest)
{
opensearchClient.ThrowIfNull(nameof(opensearchClient));
snapshotRequest.ThrowIfNull(nameof(snapshotRequest));
_opensearchClient = opensearchClient;
_snapshotRequest = snapshotRequest;
_snapshotRequest.RequestParameters.SetRequestMetaData(RequestMetaDataFactory.SnapshotHelperRequestMetaData());
_snapshotStatusHumbleObject = new SnapshotStatusHumbleObject(opensearchClient, snapshotRequest);
_snapshotStatusHumbleObject.Completed += StopTimer;
_snapshotStatusHumbleObject.Error += StopTimer;
}
private async Task ScrollToCompletionAsync(int slice, IObserver <ScrollAllResponse <T> > observer, ISearchResponse <T> searchResult)
{
var page = 0;
ThrowOnBadSearchResult(searchResult, slice, page);
var scroll = _scrollAllRequest.ScrollTime;
while (searchResult.IsValid && searchResult.Documents.HasAny())
{
if (_backPressure != null)
{
await _backPressure.WaitAsync(_compositeCancelToken).ConfigureAwait(false);
}
observer.OnNext(new ScrollAllResponse <T>()
{
Slice = slice,
SearchResponse = searchResult,
Scroll = page,
});
page++;
var request = new ScrollRequest(searchResult.ScrollId, scroll);
if (request.RequestConfiguration is null)
{
request.RequestConfiguration = new RequestConfiguration();
}
switch (_scrollAllRequest)
{
case IHelperCallable helperCallable when helperCallable.ParentMetaData is object:
request.RequestConfiguration.SetRequestMetaData(helperCallable.ParentMetaData);
break;
default:
request.RequestConfiguration.SetRequestMetaData(RequestMetaDataFactory.ScrollHelperRequestMetaData());
break;
}
searchResult = await _client.ScrollAsync <T>(request, _compositeCancelToken).ConfigureAwait(false);
ThrowOnBadSearchResult(searchResult, slice, page);
}
}
private async Task <BulkAllResponse> BulkAsync(IList <T> buffer, long page, int backOffRetries)
{
_compositeCancelToken.ThrowIfCancellationRequested();
var request = _partitionedBulkRequest;
var response = await _client.BulkAsync(s =>
{
s.Index(request.Index);
s.Timeout(request.Timeout);
if (request.BufferToBulk != null)
{
request.BufferToBulk(s, buffer);
}
else
{
s.IndexMany(buffer);
}
if (!string.IsNullOrEmpty(request.Pipeline))
{
s.Pipeline(request.Pipeline);
}
if (request.Routing != null)
{
s.Routing(request.Routing);
}
if (request.WaitForActiveShards.HasValue)
{
s.WaitForActiveShards(request.WaitForActiveShards.ToString());
}
switch (_partitionedBulkRequest)
{
case IHelperCallable helperCallable when helperCallable.ParentMetaData is object:
s.RequestConfiguration(rc => rc.RequestMetaData(helperCallable.ParentMetaData));
break;
default:
s.RequestConfiguration(rc => rc.RequestMetaData(RequestMetaDataFactory.BulkHelperRequestMetaData()));
break;
}
return(s);
}, _compositeCancelToken)
.ConfigureAwait(false);
_compositeCancelToken.ThrowIfCancellationRequested();
_bulkResponseCallback?.Invoke(response);
if (!response.ApiCall.Success)
{
return(await HandleBulkRequest(buffer, page, backOffRetries, response).ConfigureAwait(false));
}
var retryableDocuments = new List <T>();
var droppedDocuments = new List <Tuple <BulkResponseItemBase, T> >();
foreach (var documentWithResponse in response.Items.Zip(buffer, Tuple.Create))
{
if (documentWithResponse.Item1.IsValid)
{
continue;
}
if (_retryPredicate(documentWithResponse.Item1, documentWithResponse.Item2))
{
retryableDocuments.Add(documentWithResponse.Item2);
}
else
{
droppedDocuments.Add(documentWithResponse);
}
}
HandleDroppedDocuments(droppedDocuments, response);
if (retryableDocuments.Count > 0 && backOffRetries < _backOffRetries)
{
return(await RetryDocuments(page, ++backOffRetries, retryableDocuments).ConfigureAwait(false));
}
if (retryableDocuments.Count > 0)
{
throw ThrowOnBadBulk(response, $"Bulk indexing failed and after retrying {backOffRetries} times");
}
request.BackPressure?.Release();
return(new BulkAllResponse {
Retries = backOffRetries, Page = page, Items = response.Items
});
}
