protected override async Task OnNextAsync(List <Envelope <PutRecordsRequestEntry> > records, long batchBytes)
{
_logger?.LogDebug($"KinesisStreamSink {this.Id} sending {records.Count} records {batchBytes} bytes.");
DateTime utcNow = DateTime.UtcNow;
_clientLatency = (long)records.Average(r => (utcNow - r.Timestamp).TotalMilliseconds);
long elapsedMilliseconds = Utility.GetElapsedMilliseconds();
try
{
_recordsAttempted += records.Count;
_bytesAttempted += batchBytes;
var response = await _kinesisClient.PutRecordsAsync(new PutRecordsRequest()
{
StreamName = _streamName,
Records = records.Select(r => r.Data).ToList()
});
_throttle.SetSuccess();
_latency = Utility.GetElapsedMilliseconds() - elapsedMilliseconds;
_recordsSuccess += records.Count;
_logger?.LogDebug($"KinesisStreamSink {this.Id} successfully sent {records.Count} records {batchBytes} bytes.");
this.SaveBookmarks(records);
}
catch (Exception ex)
{
_latency = Utility.GetElapsedMilliseconds() - elapsedMilliseconds;
_throttle.SetError();
if (this.IsRecoverableException(ex) && _buffer.Requeue(records, _throttle.ConsecutiveErrorCount < _maxAttempts))
{
_recoverableServiceErrors++;
_recordsFailedRecoverable += records.Count;
if (LogThrottler.ShouldWrite(LogThrottler.CreateLogTypeId(this.GetType().FullName, "OnNextAsync", "Requeued", this.Id), TimeSpan.FromMinutes(5)))
{
_logger?.LogWarning($"KinesisStreamSink {this.Id} requeued request after exception (attempt {_throttle.ConsecutiveErrorCount}): {ex.ToMinimized()}");
}
}
else
{
_nonrecoverableServiceErrors++;
_recordsFailedNonrecoverable += records.Count;
_logger?.LogError($"KinesisStreamSink {this.Id} client exception after {_throttle.ConsecutiveErrorCount} attempts: {ex.ToMinimized()}");
}
}
PublishMetrics(MetricsConstants.KINESIS_STREAM_PREFIX);
}
public void TestLogThrottler()
{
int logTypeId = 1;
TimeSpan deplay = TimeSpan.FromSeconds(1);
bool shouldWrite = LogThrottler.ShouldWrite(logTypeId, deplay);
Assert.True(shouldWrite);
//Try again and should return false
shouldWrite = LogThrottler.ShouldWrite(logTypeId, deplay);
Assert.False(shouldWrite);
//Wait for 2 seconds and should return true
Thread.Sleep(2000);
shouldWrite = LogThrottler.ShouldWrite(logTypeId, deplay);
Assert.True(shouldWrite);
}
protected override async Task OnNextAsync(List <Envelope <Record> > envelopes, long batchBytes)
{
_logger?.LogDebug($"KinesisFirehoseSink {this.Id} sending {envelopes.Count} records {batchBytes} bytes.");
DateTime utcNow = DateTime.UtcNow;
_clientLatency = (long)envelopes.Average(r => (utcNow - r.Timestamp).TotalMilliseconds);
long elapsedMilliseconds = Utility.GetElapsedMilliseconds();
try
{
_recordsAttempted += envelopes.Count;
_bytesAttempted += batchBytes;
List <Record> records = envelopes.Select(r => r.Data).ToList();
if (this.CanCombineRecords)
{
records = CombineRecords(records);
}
PutRecordBatchResponse response = await _firehoseClient.PutRecordBatchAsync(_deliveryStreamName, records);
_latency = Utility.GetElapsedMilliseconds() - elapsedMilliseconds;
if (response.FailedPutCount > 0 && response.RequestResponses != null)
{
_throttle.SetError();
_recoverableServiceErrors++;
_recordsSuccess += envelopes.Count - response.FailedPutCount;
_logger?.LogError($"KinesisFirehoseSink client {this.Id} BatchRecordCount={envelopes.Count} FailedPutCount={response.FailedPutCount} Attempt={_throttle.ConsecutiveErrorCount}");
List <Envelope <Record> > requeueRecords = new List <Envelope <Record> >();
for (int i = 0; i < response.RequestResponses.Count; i++)
{
var reqResponse = response.RequestResponses[i];
if (!string.IsNullOrEmpty(reqResponse.ErrorCode))
{
requeueRecords.Add(envelopes[i]);
//When there is error, reqResponse.RecordId would be null. So we have to use the sequence number within the batch here.
if (_throttle.ConsecutiveErrorCount >= _maxAttempts)
{
_logger?.LogDebug($"Record {i} error {reqResponse.ErrorCode}: {reqResponse.ErrorMessage}");
}
}
}
if (_buffer.Requeue(requeueRecords, _throttle.ConsecutiveErrorCount < _maxAttempts))
{
_recordsFailedRecoverable += response.FailedPutCount;
}
else
{
_recordsFailedNonrecoverable += response.FailedPutCount;
throw new AmazonKinesisFirehoseException($"Messages discarded after {_throttle.ConsecutiveErrorCount} attempts.");
}
}
else
{
_throttle.SetSuccess();
_recordsSuccess += envelopes.Count;
_logger?.LogDebug($"KinesisFirehoseSink {this.Id} successfully sent {envelopes.Count} records {batchBytes} bytes.");
this.SaveBookmarks(envelopes);
}
}
catch (Exception ex)
{
_latency = Utility.GetElapsedMilliseconds() - elapsedMilliseconds;
_throttle.SetError();
if (IsRecoverableException(ex) &&
_buffer.Requeue(envelopes, _throttle.ConsecutiveErrorCount < _maxAttempts))
{
_recoverableServiceErrors++;
_recordsFailedRecoverable += envelopes.Count;
if (LogThrottler.ShouldWrite(LogThrottler.CreateLogTypeId(this.GetType().FullName, "OnNextAsync", "Requeued", this.Id), TimeSpan.FromMinutes(5)))
{
_logger?.LogWarning($"KinesisFirehoseSink {this.Id} requeued request after exception (attempt {_throttle.ConsecutiveErrorCount}): {ex.ToMinimized()}");
}
}
else
{
_nonrecoverableServiceErrors++;
_recordsFailedNonrecoverable += envelopes.Count;
_logger?.LogError($"KinesisFirehoseSink {this.Id} client exception after {_throttle.ConsecutiveErrorCount} attempts: {ex.ToMinimized()}");
}
}
PublishMetrics(MetricsConstants.KINESIS_FIREHOSE_PREFIX);
}
