private async Task <bool> TryPublishRecordsAsync(
IAmazonKinesisFirehose firehose,
List <Record> records
)
{
m_metrics.IncrementPutRecordBatchRequests();
try {
int recordCount = records.Count;
var batchRequest = new PutRecordBatchRequest {
DeliveryStreamName = m_args.DeliveryStreamName,
Records = records
};
PutRecordBatchResponse response = await firehose
.PutRecordBatchAsync(batchRequest);
LogEventInfo log = new LogEventInfo {
Properties =
{
{ "Response", response }
}
};
int failedPutCount = response.FailedPutCount;
if (failedPutCount == recordCount)
{
log.Level = LogLevel.Error;
log.Message = "Failed to put all records in batch.";
m_metrics.IncrementRecordsFailed(failedPutCount);
}
else if (failedPutCount > 0)
{
log.Level = LogLevel.Warn;
log.Message = "Failed to put some records in batch.";
m_metrics.IncrementRecordsPut(recordCount - failedPutCount);
m_metrics.IncrementRecordsFailed(failedPutCount);
}
else
{
log.Level = LogLevel.Debug;
log.Message = "Successfully put all records.";
m_metrics.IncrementRecordsPut(recordCount);
}
m_log.Log(log);
return(true);
} catch (Exception err) {
m_log.Error(err, "Failed to put record batch.");
m_metrics.IncrementPutRecordBatchErrors();
return(false);
}
}
protected override PutRecordBatchResponse SendRecords(List <Record> records, out bool successful)
{
var request = new PutRecordBatchRequest
{
DeliveryStreamName = _streamName,
Records = records
};
SelfLog.WriteLine("Writing {0} records to firehose", records.Count);
var putRecordBatchTask = _kinesisFirehoseClient.PutRecordBatchAsync(request);
successful = putRecordBatchTask.GetAwaiter().GetResult().FailedPutCount == 0;
return(putRecordBatchTask.Result);
}
public async Task <PutRecordBatchResponse> AttemptPutRecords(List <Record> kinesisRecords)
{
var retryPolicy = CreateExceptionRetryPolicy();
PutRecordBatchResponse response = null;
await retryPolicy.ExecuteAsync(async() =>
{
response = await _kinesisFirehoseClient.PutRecordBatchAsync(DeliveryStreamName, kinesisRecords);
if (response.FailedPutCount > 0)
{
await RetryFailedRecords(response, kinesisRecords);
}
});
return(response);
}
private Amazon.KinesisFirehose.Model.PutRecordBatchResponse CallAWSServiceOperation(IAmazonKinesisFirehose client, Amazon.KinesisFirehose.Model.PutRecordBatchRequest request)
{
Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Kinesis Firehose", "PutRecordBatch");
try
{
#if DESKTOP
return(client.PutRecordBatch(request));
#elif CORECLR
return(client.PutRecordBatchAsync(request).GetAwaiter().GetResult());
#else
#error "Unknown build edition"
#endif
}
catch (AmazonServiceException exc)
{
var webException = exc.InnerException as System.Net.WebException;
if (webException != null)
{
throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException);
}
throw;
}
}
public async Task <bool> RecordAsync <T>(IEnumerable <T> events, string streamName)
{
if (!events.Any())
{
return(false);
}
var records = new List <Record>();
foreach (T obj in events)
{
string objAsJson = _serializer.Serialize(events);
byte[] objAsBytes = Encoding.UTF8.GetBytes(objAsJson + "\n");
using MemoryStream ms = new MemoryStream(objAsBytes);
Record record = new Record {
Data = ms
};
records.Add(record);
}
var request = new PutRecordBatchRequest
{
DeliveryStreamName = streamName,
Records = records
};
PutRecordBatchResponse response = await _client.PutRecordBatchAsync(request);
if (response.HttpStatusCode != HttpStatusCode.OK)
{
throw new System.Exception($"Error sending message. HttpStatusCode: {response.HttpStatusCode}");
}
return(true);
}
protected override async Task OnNextAsync(List <Envelope <Record> > envelopes, long batchBytes)
{
_logger?.LogDebug($"KinesisFirehoseSink {this.Id} sending {envelopes.Count} records {batchBytes} bytes.");
DateTime utcNow = DateTime.UtcNow;
_clientLatency = (long)envelopes.Average(r => (utcNow - r.Timestamp).TotalMilliseconds);
long elapsedMilliseconds = Utility.GetElapsedMilliseconds();
try
{
_recordsAttempted += envelopes.Count;
_bytesAttempted += batchBytes;
List <Record> records = envelopes.Select(r => r.Data).ToList();
if (this.CanCombineRecords)
{
records = CombineRecords(records);
}
PutRecordBatchResponse response = await _firehoseClient.PutRecordBatchAsync(_deliveryStreamName, records);
_latency = Utility.GetElapsedMilliseconds() - elapsedMilliseconds;
if (response.FailedPutCount > 0 && response.RequestResponses != null)
{
_throttle.SetError();
_recoverableServiceErrors++;
_recordsSuccess += envelopes.Count - response.FailedPutCount;
_logger?.LogError($"KinesisFirehoseSink client {this.Id} BatchRecordCount={envelopes.Count} FailedPutCount={response.FailedPutCount} Attempt={_throttle.ConsecutiveErrorCount}");
List <Envelope <Record> > requeueRecords = new List <Envelope <Record> >();
for (int i = 0; i < response.RequestResponses.Count; i++)
{
var reqResponse = response.RequestResponses[i];
if (!string.IsNullOrEmpty(reqResponse.ErrorCode))
{
requeueRecords.Add(envelopes[i]);
//When there is error, reqResponse.RecordId would be null. So we have to use the sequence number within the batch here.
if (_throttle.ConsecutiveErrorCount >= _maxAttempts)
{
_logger?.LogDebug($"Record {i} error {reqResponse.ErrorCode}: {reqResponse.ErrorMessage}");
}
}
}
if (_buffer.Requeue(requeueRecords, _throttle.ConsecutiveErrorCount < _maxAttempts))
{
_recordsFailedRecoverable += response.FailedPutCount;
}
else
{
_recordsFailedNonrecoverable += response.FailedPutCount;
throw new AmazonKinesisFirehoseException($"Messages discarded after {_throttle.ConsecutiveErrorCount} attempts.");
}
}
else
{
_throttle.SetSuccess();
_recordsSuccess += envelopes.Count;
_logger?.LogDebug($"KinesisFirehoseSink {this.Id} successfully sent {envelopes.Count} records {batchBytes} bytes.");
this.SaveBookmarks(envelopes);
}
}
catch (Exception ex)
{
_latency = Utility.GetElapsedMilliseconds() - elapsedMilliseconds;
_throttle.SetError();
if (IsRecoverableException(ex) &&
_buffer.Requeue(envelopes, _throttle.ConsecutiveErrorCount < _maxAttempts))
{
_recoverableServiceErrors++;
_recordsFailedRecoverable += envelopes.Count;
if (LogThrottler.ShouldWrite(LogThrottler.CreateLogTypeId(this.GetType().FullName, "OnNextAsync", "Requeued", this.Id), TimeSpan.FromMinutes(5)))
{
_logger?.LogWarning($"KinesisFirehoseSink {this.Id} requeued request after exception (attempt {_throttle.ConsecutiveErrorCount}): {ex.ToMinimized()}");
}
}
else
{
_nonrecoverableServiceErrors++;
_recordsFailedNonrecoverable += envelopes.Count;
_logger?.LogError($"KinesisFirehoseSink {this.Id} client exception after {_throttle.ConsecutiveErrorCount} attempts: {ex.ToMinimized()}");
}
}
PublishMetrics(MetricsConstants.KINESIS_FIREHOSE_PREFIX);
}
