private async Task <bool> TryPublishRecordsAsync( IAmazonKinesisFirehose firehose, List <Record> records ) { m_metrics.IncrementPutRecordBatchRequests(); try { int recordCount = records.Count; var batchRequest = new PutRecordBatchRequest { DeliveryStreamName = m_args.DeliveryStreamName, Records = records }; PutRecordBatchResponse response = await firehose .PutRecordBatchAsync(batchRequest); LogEventInfo log = new LogEventInfo { Properties = { { "Response", response } } }; int failedPutCount = response.FailedPutCount; if (failedPutCount == recordCount) { log.Level = LogLevel.Error; log.Message = "Failed to put all records in batch."; m_metrics.IncrementRecordsFailed(failedPutCount); } else if (failedPutCount > 0) { log.Level = LogLevel.Warn; log.Message = "Failed to put some records in batch."; m_metrics.IncrementRecordsPut(recordCount - failedPutCount); m_metrics.IncrementRecordsFailed(failedPutCount); } else { log.Level = LogLevel.Debug; log.Message = "Successfully put all records."; m_metrics.IncrementRecordsPut(recordCount); } m_log.Log(log); return(true); } catch (Exception err) { m_log.Error(err, "Failed to put record batch."); m_metrics.IncrementPutRecordBatchErrors(); return(false); } }
protected override PutRecordBatchResponse SendRecords(List <Record> records, out bool successful) { var request = new PutRecordBatchRequest { DeliveryStreamName = _streamName, Records = records }; SelfLog.WriteLine("Writing {0} records to firehose", records.Count); var putRecordBatchTask = _kinesisFirehoseClient.PutRecordBatchAsync(request); successful = putRecordBatchTask.GetAwaiter().GetResult().FailedPutCount == 0; return(putRecordBatchTask.Result); }
public async Task <PutRecordBatchResponse> AttemptPutRecords(List <Record> kinesisRecords) { var retryPolicy = CreateExceptionRetryPolicy(); PutRecordBatchResponse response = null; await retryPolicy.ExecuteAsync(async() => { response = await _kinesisFirehoseClient.PutRecordBatchAsync(DeliveryStreamName, kinesisRecords); if (response.FailedPutCount > 0) { await RetryFailedRecords(response, kinesisRecords); } }); return(response); }
private Amazon.KinesisFirehose.Model.PutRecordBatchResponse CallAWSServiceOperation(IAmazonKinesisFirehose client, Amazon.KinesisFirehose.Model.PutRecordBatchRequest request) { Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Kinesis Firehose", "PutRecordBatch"); try { #if DESKTOP return(client.PutRecordBatch(request)); #elif CORECLR return(client.PutRecordBatchAsync(request).GetAwaiter().GetResult()); #else #error "Unknown build edition" #endif } catch (AmazonServiceException exc) { var webException = exc.InnerException as System.Net.WebException; if (webException != null) { throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException); } throw; } }
public async Task <bool> RecordAsync <T>(IEnumerable <T> events, string streamName) { if (!events.Any()) { return(false); } var records = new List <Record>(); foreach (T obj in events) { string objAsJson = _serializer.Serialize(events); byte[] objAsBytes = Encoding.UTF8.GetBytes(objAsJson + "\n"); using MemoryStream ms = new MemoryStream(objAsBytes); Record record = new Record { Data = ms }; records.Add(record); } var request = new PutRecordBatchRequest { DeliveryStreamName = streamName, Records = records }; PutRecordBatchResponse response = await _client.PutRecordBatchAsync(request); if (response.HttpStatusCode != HttpStatusCode.OK) { throw new System.Exception($"Error sending message. HttpStatusCode: {response.HttpStatusCode}"); } return(true); }
protected override async Task OnNextAsync(List <Envelope <Record> > envelopes, long batchBytes) { _logger?.LogDebug($"KinesisFirehoseSink {this.Id} sending {envelopes.Count} records {batchBytes} bytes."); DateTime utcNow = DateTime.UtcNow; _clientLatency = (long)envelopes.Average(r => (utcNow - r.Timestamp).TotalMilliseconds); long elapsedMilliseconds = Utility.GetElapsedMilliseconds(); try { _recordsAttempted += envelopes.Count; _bytesAttempted += batchBytes; List <Record> records = envelopes.Select(r => r.Data).ToList(); if (this.CanCombineRecords) { records = CombineRecords(records); } PutRecordBatchResponse response = await _firehoseClient.PutRecordBatchAsync(_deliveryStreamName, records); _latency = Utility.GetElapsedMilliseconds() - elapsedMilliseconds; if (response.FailedPutCount > 0 && response.RequestResponses != null) { _throttle.SetError(); _recoverableServiceErrors++; _recordsSuccess += envelopes.Count - response.FailedPutCount; _logger?.LogError($"KinesisFirehoseSink client {this.Id} BatchRecordCount={envelopes.Count} FailedPutCount={response.FailedPutCount} Attempt={_throttle.ConsecutiveErrorCount}"); List <Envelope <Record> > requeueRecords = new List <Envelope <Record> >(); for (int i = 0; i < response.RequestResponses.Count; i++) { var reqResponse = response.RequestResponses[i]; if (!string.IsNullOrEmpty(reqResponse.ErrorCode)) { requeueRecords.Add(envelopes[i]); //When there is error, reqResponse.RecordId would be null. So we have to use the sequence number within the batch here. if (_throttle.ConsecutiveErrorCount >= _maxAttempts) { _logger?.LogDebug($"Record {i} error {reqResponse.ErrorCode}: {reqResponse.ErrorMessage}"); } } } if (_buffer.Requeue(requeueRecords, _throttle.ConsecutiveErrorCount < _maxAttempts)) { _recordsFailedRecoverable += response.FailedPutCount; } else { _recordsFailedNonrecoverable += response.FailedPutCount; throw new AmazonKinesisFirehoseException($"Messages discarded after {_throttle.ConsecutiveErrorCount} attempts."); } } else { _throttle.SetSuccess(); _recordsSuccess += envelopes.Count; _logger?.LogDebug($"KinesisFirehoseSink {this.Id} successfully sent {envelopes.Count} records {batchBytes} bytes."); this.SaveBookmarks(envelopes); } } catch (Exception ex) { _latency = Utility.GetElapsedMilliseconds() - elapsedMilliseconds; _throttle.SetError(); if (IsRecoverableException(ex) && _buffer.Requeue(envelopes, _throttle.ConsecutiveErrorCount < _maxAttempts)) { _recoverableServiceErrors++; _recordsFailedRecoverable += envelopes.Count; if (LogThrottler.ShouldWrite(LogThrottler.CreateLogTypeId(this.GetType().FullName, "OnNextAsync", "Requeued", this.Id), TimeSpan.FromMinutes(5))) { _logger?.LogWarning($"KinesisFirehoseSink {this.Id} requeued request after exception (attempt {_throttle.ConsecutiveErrorCount}): {ex.ToMinimized()}"); } } else { _nonrecoverableServiceErrors++; _recordsFailedNonrecoverable += envelopes.Count; _logger?.LogError($"KinesisFirehoseSink {this.Id} client exception after {_throttle.ConsecutiveErrorCount} attempts: {ex.ToMinimized()}"); } } PublishMetrics(MetricsConstants.KINESIS_FIREHOSE_PREFIX); }