protected override async Task OnNextAsync(List <Envelope <PutRecordsRequestEntry> > records, long batchBytes) { _logger?.LogDebug($"KinesisStreamSink {this.Id} sending {records.Count} records {batchBytes} bytes."); DateTime utcNow = DateTime.UtcNow; _clientLatency = (long)records.Average(r => (utcNow - r.Timestamp).TotalMilliseconds); long elapsedMilliseconds = Utility.GetElapsedMilliseconds(); try { _recordsAttempted += records.Count; _bytesAttempted += batchBytes; var response = await _kinesisClient.PutRecordsAsync(new PutRecordsRequest() { StreamName = _streamName, Records = records.Select(r => r.Data).ToList() }); _throttle.SetSuccess(); _latency = Utility.GetElapsedMilliseconds() - elapsedMilliseconds; _recordsSuccess += records.Count; _logger?.LogDebug($"KinesisStreamSink {this.Id} successfully sent {records.Count} records {batchBytes} bytes."); this.SaveBookmarks(records); } catch (Exception ex) { _latency = Utility.GetElapsedMilliseconds() - elapsedMilliseconds; _throttle.SetError(); if (this.IsRecoverableException(ex) && _buffer.Requeue(records, _throttle.ConsecutiveErrorCount < _maxAttempts)) { _recoverableServiceErrors++; _recordsFailedRecoverable += records.Count; if (LogThrottler.ShouldWrite(LogThrottler.CreateLogTypeId(this.GetType().FullName, "OnNextAsync", "Requeued", this.Id), TimeSpan.FromMinutes(5))) { _logger?.LogWarning($"KinesisStreamSink {this.Id} requeued request after exception (attempt {_throttle.ConsecutiveErrorCount}): {ex.ToMinimized()}"); } } else { _nonrecoverableServiceErrors++; _recordsFailedNonrecoverable += records.Count; _logger?.LogError($"KinesisStreamSink {this.Id} client exception after {_throttle.ConsecutiveErrorCount} attempts: {ex.ToMinimized()}"); } } PublishMetrics(MetricsConstants.KINESIS_STREAM_PREFIX); }
public void TestLogThrottler() { int logTypeId = 1; TimeSpan deplay = TimeSpan.FromSeconds(1); bool shouldWrite = LogThrottler.ShouldWrite(logTypeId, deplay); Assert.True(shouldWrite); //Try again and should return false shouldWrite = LogThrottler.ShouldWrite(logTypeId, deplay); Assert.False(shouldWrite); //Wait for 2 seconds and should return true Thread.Sleep(2000); shouldWrite = LogThrottler.ShouldWrite(logTypeId, deplay); Assert.True(shouldWrite); }
protected override async Task OnNextAsync(List <Envelope <Record> > envelopes, long batchBytes) { _logger?.LogDebug($"KinesisFirehoseSink {this.Id} sending {envelopes.Count} records {batchBytes} bytes."); DateTime utcNow = DateTime.UtcNow; _clientLatency = (long)envelopes.Average(r => (utcNow - r.Timestamp).TotalMilliseconds); long elapsedMilliseconds = Utility.GetElapsedMilliseconds(); try { _recordsAttempted += envelopes.Count; _bytesAttempted += batchBytes; List <Record> records = envelopes.Select(r => r.Data).ToList(); if (this.CanCombineRecords) { records = CombineRecords(records); } PutRecordBatchResponse response = await _firehoseClient.PutRecordBatchAsync(_deliveryStreamName, records); _latency = Utility.GetElapsedMilliseconds() - elapsedMilliseconds; if (response.FailedPutCount > 0 && response.RequestResponses != null) { _throttle.SetError(); _recoverableServiceErrors++; _recordsSuccess += envelopes.Count - response.FailedPutCount; _logger?.LogError($"KinesisFirehoseSink client {this.Id} BatchRecordCount={envelopes.Count} FailedPutCount={response.FailedPutCount} Attempt={_throttle.ConsecutiveErrorCount}"); List <Envelope <Record> > requeueRecords = new List <Envelope <Record> >(); for (int i = 0; i < response.RequestResponses.Count; i++) { var reqResponse = response.RequestResponses[i]; if (!string.IsNullOrEmpty(reqResponse.ErrorCode)) { requeueRecords.Add(envelopes[i]); //When there is error, reqResponse.RecordId would be null. So we have to use the sequence number within the batch here. if (_throttle.ConsecutiveErrorCount >= _maxAttempts) { _logger?.LogDebug($"Record {i} error {reqResponse.ErrorCode}: {reqResponse.ErrorMessage}"); } } } if (_buffer.Requeue(requeueRecords, _throttle.ConsecutiveErrorCount < _maxAttempts)) { _recordsFailedRecoverable += response.FailedPutCount; } else { _recordsFailedNonrecoverable += response.FailedPutCount; throw new AmazonKinesisFirehoseException($"Messages discarded after {_throttle.ConsecutiveErrorCount} attempts."); } } else { _throttle.SetSuccess(); _recordsSuccess += envelopes.Count; _logger?.LogDebug($"KinesisFirehoseSink {this.Id} successfully sent {envelopes.Count} records {batchBytes} bytes."); this.SaveBookmarks(envelopes); } } catch (Exception ex) { _latency = Utility.GetElapsedMilliseconds() - elapsedMilliseconds; _throttle.SetError(); if (IsRecoverableException(ex) && _buffer.Requeue(envelopes, _throttle.ConsecutiveErrorCount < _maxAttempts)) { _recoverableServiceErrors++; _recordsFailedRecoverable += envelopes.Count; if (LogThrottler.ShouldWrite(LogThrottler.CreateLogTypeId(this.GetType().FullName, "OnNextAsync", "Requeued", this.Id), TimeSpan.FromMinutes(5))) { _logger?.LogWarning($"KinesisFirehoseSink {this.Id} requeued request after exception (attempt {_throttle.ConsecutiveErrorCount}): {ex.ToMinimized()}"); } } else { _nonrecoverableServiceErrors++; _recordsFailedNonrecoverable += envelopes.Count; _logger?.LogError($"KinesisFirehoseSink {this.Id} client exception after {_throttle.ConsecutiveErrorCount} attempts: {ex.ToMinimized()}"); } } PublishMetrics(MetricsConstants.KINESIS_FIREHOSE_PREFIX); }