public async Task RunAsync() { // TODO start from the last processed page // make sure the ingestion eventually stops (e.g. if a systematic error prevents us from getting NotFoundException) const int PAGE_LIMIT = 99999; for (int page = 0; page < PAGE_LIMIT; page++) { IngestionResult result = await IngestBatch(page); switch (result) { case IngestionResult.Success: // continue to the next page break; case IngestionResult.NothingToProcess: // it's the last batch, finish _logger.LogInformation("Finished processing all batches"); return; case IngestionResult.Failure: // assuming the error is not transient, we can only continue break; } } _logger.LogError("Stopped ingestion as error count exceeded the limit"); }
public override async Task <IngestionResult> TryIngestAsync(string payload, string mediaType) { if (IsDisposed) { throw new ObjectDisposedException(nameof(TestIngestionApi)); } var ingestionPayload = new IngestionPayload(payload, mediaType); IngestionResult result; if (_onIngestAsync != null) { result = await _onIngestAsync(ingestionPayload); } else { result = new IngestionResult(true, HttpStatusCode.Accepted, null); } if (result.Succeeded) { if (!_ingested.Writer.TryWrite(ingestionPayload)) { throw new InvalidOperationException("Channel capacity exceeded."); } } return(result); }