public ScrollAllObservable(IOpenSearchClient client, IScrollAllRequest scrollAllRequest, CancellationToken cancellationToken = default) { _scrollAllRequest = scrollAllRequest; _searchRequest = scrollAllRequest?.Search ?? new SearchRequest <T>(); switch (_scrollAllRequest) { case IHelperCallable helperCallable when helperCallable.ParentMetaData is object: _searchRequest.RequestParameters.SetRequestMetaData(helperCallable.ParentMetaData); break; default: _searchRequest.RequestParameters.SetRequestMetaData(RequestMetaDataFactory.ScrollHelperRequestMetaData()); break; } if (_searchRequest.Sort == null) { _searchRequest.Sort = FieldSort.ByDocumentOrder; } _searchRequest.RequestParameters.Scroll = _scrollAllRequest.ScrollTime.ToTimeSpan(); _client = client; _compositeCancelTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); _compositeCancelToken = _compositeCancelTokenSource.Token; _backPressure = _scrollAllRequest.BackPressure; }
private void RefreshOnCompleted() { if (!_partitionedBulkRequest.RefreshOnCompleted) { return; } var indices = _partitionedBulkRequest.RefreshIndices ?? _partitionedBulkRequest.Index; if (indices == null) { return; } var refresh = _client.Indices.Refresh(indices, r => r.RequestConfiguration(rc => { switch (_partitionedBulkRequest) { case IHelperCallable helperCallable when helperCallable.ParentMetaData is object: rc.RequestMetaData(helperCallable.ParentMetaData); break; default: rc.RequestMetaData(RequestMetaDataFactory.BulkHelperRequestMetaData()); break; } return(rc); })); if (!refresh.IsValid) { throw Throw($"Refreshing after all documents have indexed failed", refresh.ApiCall); } }
public SnapshotObservable(IOpenSearchClient opensearchClient, ISnapshotRequest snapshotRequest) { opensearchClient.ThrowIfNull(nameof(opensearchClient)); snapshotRequest.ThrowIfNull(nameof(snapshotRequest)); _opensearchClient = opensearchClient; _snapshotRequest = snapshotRequest; _snapshotRequest.RequestParameters.SetRequestMetaData(RequestMetaDataFactory.SnapshotHelperRequestMetaData()); _snapshotStatusHumbleObject = new SnapshotStatusHumbleObject(opensearchClient, snapshotRequest); _snapshotStatusHumbleObject.Completed += StopTimer; _snapshotStatusHumbleObject.Error += StopTimer; }
private async Task ScrollToCompletionAsync(int slice, IObserver <ScrollAllResponse <T> > observer, ISearchResponse <T> searchResult) { var page = 0; ThrowOnBadSearchResult(searchResult, slice, page); var scroll = _scrollAllRequest.ScrollTime; while (searchResult.IsValid && searchResult.Documents.HasAny()) { if (_backPressure != null) { await _backPressure.WaitAsync(_compositeCancelToken).ConfigureAwait(false); } observer.OnNext(new ScrollAllResponse <T>() { Slice = slice, SearchResponse = searchResult, Scroll = page, }); page++; var request = new ScrollRequest(searchResult.ScrollId, scroll); if (request.RequestConfiguration is null) { request.RequestConfiguration = new RequestConfiguration(); } switch (_scrollAllRequest) { case IHelperCallable helperCallable when helperCallable.ParentMetaData is object: request.RequestConfiguration.SetRequestMetaData(helperCallable.ParentMetaData); break; default: request.RequestConfiguration.SetRequestMetaData(RequestMetaDataFactory.ScrollHelperRequestMetaData()); break; } searchResult = await _client.ScrollAsync <T>(request, _compositeCancelToken).ConfigureAwait(false); ThrowOnBadSearchResult(searchResult, slice, page); } }
private async Task <BulkAllResponse> BulkAsync(IList <T> buffer, long page, int backOffRetries) { _compositeCancelToken.ThrowIfCancellationRequested(); var request = _partitionedBulkRequest; var response = await _client.BulkAsync(s => { s.Index(request.Index); s.Timeout(request.Timeout); if (request.BufferToBulk != null) { request.BufferToBulk(s, buffer); } else { s.IndexMany(buffer); } if (!string.IsNullOrEmpty(request.Pipeline)) { s.Pipeline(request.Pipeline); } if (request.Routing != null) { s.Routing(request.Routing); } if (request.WaitForActiveShards.HasValue) { s.WaitForActiveShards(request.WaitForActiveShards.ToString()); } switch (_partitionedBulkRequest) { case IHelperCallable helperCallable when helperCallable.ParentMetaData is object: s.RequestConfiguration(rc => rc.RequestMetaData(helperCallable.ParentMetaData)); break; default: s.RequestConfiguration(rc => rc.RequestMetaData(RequestMetaDataFactory.BulkHelperRequestMetaData())); break; } return(s); }, _compositeCancelToken) .ConfigureAwait(false); _compositeCancelToken.ThrowIfCancellationRequested(); _bulkResponseCallback?.Invoke(response); if (!response.ApiCall.Success) { return(await HandleBulkRequest(buffer, page, backOffRetries, response).ConfigureAwait(false)); } var retryableDocuments = new List <T>(); var droppedDocuments = new List <Tuple <BulkResponseItemBase, T> >(); foreach (var documentWithResponse in response.Items.Zip(buffer, Tuple.Create)) { if (documentWithResponse.Item1.IsValid) { continue; } if (_retryPredicate(documentWithResponse.Item1, documentWithResponse.Item2)) { retryableDocuments.Add(documentWithResponse.Item2); } else { droppedDocuments.Add(documentWithResponse); } } HandleDroppedDocuments(droppedDocuments, response); if (retryableDocuments.Count > 0 && backOffRetries < _backOffRetries) { return(await RetryDocuments(page, ++backOffRetries, retryableDocuments).ConfigureAwait(false)); } if (retryableDocuments.Count > 0) { throw ThrowOnBadBulk(response, $"Bulk indexing failed and after retrying {backOffRetries} times"); } request.BackPressure?.Release(); return(new BulkAllResponse { Retries = backOffRetries, Page = page, Items = response.Items }); }