internal virtual PutRecordBatchResponse PutRecordBatch(PutRecordBatchRequest request)
{
var marshaller = new PutRecordBatchRequestMarshaller();
var unmarshaller = PutRecordBatchResponseUnmarshaller.Instance;
return(Invoke <PutRecordBatchRequest, PutRecordBatchResponse>(request, marshaller, unmarshaller));
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecordBatch operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecordBatch operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/firehose-2015-08-04/PutRecordBatch">REST API Reference for PutRecordBatch Operation</seealso>
public virtual Task <PutRecordBatchResponse> PutRecordBatchAsync(PutRecordBatchRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new PutRecordBatchRequestMarshaller();
var unmarshaller = PutRecordBatchResponseUnmarshaller.Instance;
return(InvokeAsync <PutRecordBatchRequest, PutRecordBatchResponse>(request, marshaller,
unmarshaller, cancellationToken));
}
/// <summary>
/// Writes multiple data records into a delivery stream in a single call, which can achieve
/// higher throughput per producer than when writing single records. To write single data
/// records into a delivery stream, use <a>PutRecord</a>. Applications using these operations
/// are referred to as producers.
///
///
/// <para>
/// By default, each delivery stream can take in up to 2,000 transactions per second,
/// 5,000 records per second, or 5 MB per second. If you use <a>PutRecord</a> and <a>PutRecordBatch</a>,
/// the limits are an aggregate across these two operations for each delivery stream.
/// For more information about limits, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Firehose Limits</a>.
/// </para>
///
/// <para>
/// Each <a>PutRecordBatch</a> request supports up to 500 records. Each record in the
/// request can be as large as 1,000 KB (before 64-bit encoding), up to a limit of 4 MB
/// for the entire request. These limits cannot be changed.
/// </para>
///
/// <para>
/// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>.
/// The data record consists of a data blob that can be up to 1,000 KB in size, and any
/// kind of data. For example, it could be a segment from a log file, geographic location
/// data, web site clickstream data, and so on.
/// </para>
///
/// <para>
/// Kinesis Firehose buffers records before delivering them to the destination. To disambiguate
/// the data blobs at the destination, a common solution is to use delimiters in the data,
/// such as a newline (<code>\n</code>) or some other character unique within the data.
/// This allows the consumer application to parse individual data items when reading the
/// data from the destination.
/// </para>
///
/// <para>
/// The <a>PutRecordBatch</a> response includes a count of failed records, <b>FailedPutCount</b>,
/// and an array of responses, <b>RequestResponses</b>. Each entry in the <b>RequestResponses</b>
/// array provides additional information about the processed record. It directly correlates
/// with a record in the request array using the same ordering, from the top to the bottom.
/// The response array always includes the same number of records as the request array.
/// <b>RequestResponses</b> includes both successfully and unsuccessfully processed records.
/// Kinesis Firehose attempts to process all records in each <a>PutRecordBatch</a> request.
/// A single record failure does not stop the processing of subsequent records.
/// </para>
///
/// <para>
/// A successfully processed record includes a <b>RecordId</b> value, which is unique
/// for the record. An unsuccessfully processed record includes <b>ErrorCode</b> and <b>ErrorMessage</b>
/// values. <b>ErrorCode</b> reflects the type of error, and is one of the following values:
/// <code>ServiceUnavailable</code> or <code>InternalFailure</code>. <b>ErrorMessage</b>
/// provides more detailed information about the error.
/// </para>
///
/// <para>
/// If there is an internal server error or a timeout, the write might have completed
/// or it might have failed. If <b>FailedPutCount</b> is greater than 0, retry the request,
/// resending only those records that might have failed processing. This minimizes the
/// possible duplicate records and also reduces the total bytes sent (and corresponding
/// charges). We recommend that you handle any duplicates at the destination.
/// </para>
///
/// <para>
/// If <a>PutRecordBatch</a> throws <b>ServiceUnavailableException</b>, back off and retry.
/// If the exception persists, it is possible that the throughput limits have been exceeded
/// for the delivery stream.
/// </para>
///
/// <para>
/// Data records sent to Kinesis Firehose are stored for 24 hours from the time they are
/// added to a delivery stream as it attempts to send the records to the destination.
/// If the destination is unreachable for more than 24 hours, the data is no longer available.
/// </para>
/// </summary>
/// <param name="deliveryStreamName">The name of the delivery stream.</param>
/// <param name="records">One or more records.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
///
/// <returns>The response from the PutRecordBatch service method, as returned by KinesisFirehose.</returns>
/// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException">
/// The specified input parameter has a value that is not valid.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException">
/// The specified resource could not be found.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException">
/// The service is unavailable, back off and retry the operation. If you continue to see
/// the exception, throughput limits for the delivery stream may have been exceeded. For
/// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Firehose Limits</a>.
/// </exception>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/firehose-2015-08-04/PutRecordBatch">REST API Reference for PutRecordBatch Operation</seealso>
public virtual Task <PutRecordBatchResponse> PutRecordBatchAsync(string deliveryStreamName, List <Record> records, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var request = new PutRecordBatchRequest();
request.DeliveryStreamName = deliveryStreamName;
request.Records = records;
return(PutRecordBatchAsync(request, cancellationToken));
}
public PutRecordBatchResponse PutRecordBatch(string deliveryStreamName, List <Record> records)
{
var request = new PutRecordBatchRequest();
request.DeliveryStreamName = deliveryStreamName;
request.Records = records;
return(PutRecordBatch(request));
}
/// <summary>
/// Writes multiple data records into a delivery stream in a single call, which can achieve
/// higher throughput per producer than when writing single records. To write single data
/// records into a delivery stream, use <a>PutRecord</a>. Applications using these operations
/// are referred to as producers.
///
///
/// <para>
/// By default, each delivery stream can take in up to 2,000 transactions per second,
/// 5,000 records per second, or 5 MB per second. If you use <a>PutRecord</a> and <a>PutRecordBatch</a>,
/// the limits are an aggregate across these two operations for each delivery stream.
/// For more information about limits, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Data Firehose Limits</a>.
/// </para>
///
/// <para>
/// Each <a>PutRecordBatch</a> request supports up to 500 records. Each record in the
/// request can be as large as 1,000 KB (before 64-bit encoding), up to a limit of 4 MB
/// for the entire request. These limits cannot be changed.
/// </para>
///
/// <para>
/// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>.
/// The data record consists of a data blob that can be up to 1,000 KB in size, and any
/// kind of data. For example, it could be a segment from a log file, geographic location
/// data, website clickstream data, and so on.
/// </para>
///
/// <para>
/// Kinesis Data Firehose buffers records before delivering them to the destination. To
/// disambiguate the data blobs at the destination, a common solution is to use delimiters
/// in the data, such as a newline (<code>\n</code>) or some other character unique within
/// the data. This allows the consumer application to parse individual data items when
/// reading the data from the destination.
/// </para>
///
/// <para>
/// The <a>PutRecordBatch</a> response includes a count of failed records, <b>FailedPutCount</b>,
/// and an array of responses, <b>RequestResponses</b>. Even if the <a>PutRecordBatch</a>
/// call succeeds, the value of <b>FailedPutCount</b> may be greater than 0, indicating
/// that there are records for which the operation didn't succeed. Each entry in the <b>RequestResponses</b>
/// array provides additional information about the processed record. It directly correlates
/// with a record in the request array using the same ordering, from the top to the bottom.
/// The response array always includes the same number of records as the request array.
/// <b>RequestResponses</b> includes both successfully and unsuccessfully processed records.
/// Kinesis Data Firehose tries to process all records in each <a>PutRecordBatch</a> request.
/// A single record failure does not stop the processing of subsequent records.
/// </para>
///
/// <para>
/// A successfully processed record includes a <b>RecordId</b> value, which is unique
/// for the record. An unsuccessfully processed record includes <b>ErrorCode</b> and <b>ErrorMessage</b>
/// values. <b>ErrorCode</b> reflects the type of error, and is one of the following values:
/// <code>ServiceUnavailableException</code> or <code>InternalFailure</code>. <b>ErrorMessage</b>
/// provides more detailed information about the error.
/// </para>
///
/// <para>
/// If there is an internal server error or a timeout, the write might have completed
/// or it might have failed. If <b>FailedPutCount</b> is greater than 0, retry the request,
/// resending only those records that might have failed processing. This minimizes the
/// possible duplicate records and also reduces the total bytes sent (and corresponding
/// charges). We recommend that you handle any duplicates at the destination.
/// </para>
///
/// <para>
/// If <a>PutRecordBatch</a> throws <b>ServiceUnavailableException</b>, back off and retry.
/// If the exception persists, it is possible that the throughput limits have been exceeded
/// for the delivery stream.
/// </para>
///
/// <para>
/// Data records sent to Kinesis Data Firehose are stored for 24 hours from the time they
/// are added to a delivery stream as it attempts to send the records to the destination.
/// If the destination is unreachable for more than 24 hours, the data is no longer available.
/// </para>
/// <important>
/// <para>
/// Don't concatenate two or more base64 strings to form the data fields of your records.
/// Instead, concatenate the raw data, then perform base64 encoding.
/// </para>
/// </important>
/// </summary>
/// <param name="deliveryStreamName">The name of the delivery stream.</param>
/// <param name="records">One or more records.</param>
/// <param name="callback">An Action delegate that is invoked when the operation completes.</param>
/// <param name="options">
/// A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.
/// </param>
///
/// <returns>The response from the PutRecordBatch service method, as returned by KinesisFirehose.</returns>
/// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException">
/// The specified input parameter has a value that is not valid.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException">
/// The specified resource could not be found.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException">
/// The service is unavailable. Back off and retry the operation. If you continue to see
/// the exception, throughput limits for the delivery stream may have been exceeded. For
/// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Data Firehose Limits</a>.
/// </exception>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/firehose-2015-08-04/PutRecordBatch">REST API Reference for PutRecordBatch Operation</seealso>
public virtual void PutRecordBatchAsync(string deliveryStreamName, List <Record> records, AmazonServiceCallback <PutRecordBatchRequest, PutRecordBatchResponse> callback, AsyncOptions options = null)
{
var request = new PutRecordBatchRequest();
request.DeliveryStreamName = deliveryStreamName;
request.Records = records;
PutRecordBatchAsync(request, callback, options);
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecordBatch operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecordBatch operation on AmazonKinesisFirehoseClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndPutRecordBatch
/// operation.</returns>
public IAsyncResult BeginPutRecordBatch(PutRecordBatchRequest request, AsyncCallback callback, object state)
{
var marshaller = new PutRecordBatchRequestMarshaller();
var unmarshaller = PutRecordBatchResponseUnmarshaller.Instance;
return(BeginInvoke <PutRecordBatchRequest>(request, marshaller, unmarshaller,
callback, state));
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecordBatch operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecordBatch operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/firehose-2015-08-04/PutRecordBatch">REST API Reference for PutRecordBatch Operation</seealso>
public virtual Task <PutRecordBatchResponse> PutRecordBatchAsync(PutRecordBatchRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var options = new InvokeOptions();
options.RequestMarshaller = PutRecordBatchRequestMarshaller.Instance;
options.ResponseUnmarshaller = PutRecordBatchResponseUnmarshaller.Instance;
return(InvokeAsync <PutRecordBatchResponse>(request, options, cancellationToken));
}
internal virtual PutRecordBatchResponse PutRecordBatch(PutRecordBatchRequest request)
{
var options = new InvokeOptions();
options.RequestMarshaller = PutRecordBatchRequestMarshaller.Instance;
options.ResponseUnmarshaller = PutRecordBatchResponseUnmarshaller.Instance;
return(Invoke <PutRecordBatchResponse>(request, options));
}
private async Task <bool> TryPublishRecordsAsync(
IAmazonKinesisFirehose firehose,
List <Record> records
)
{
m_metrics.IncrementPutRecordBatchRequests();
try {
int recordCount = records.Count;
var batchRequest = new PutRecordBatchRequest {
DeliveryStreamName = m_args.DeliveryStreamName,
Records = records
};
PutRecordBatchResponse response = await firehose
.PutRecordBatchAsync(batchRequest);
LogEventInfo log = new LogEventInfo {
Properties =
{
{ "Response", response }
}
};
int failedPutCount = response.FailedPutCount;
if (failedPutCount == recordCount)
{
log.Level = LogLevel.Error;
log.Message = "Failed to put all records in batch.";
m_metrics.IncrementRecordsFailed(failedPutCount);
}
else if (failedPutCount > 0)
{
log.Level = LogLevel.Warn;
log.Message = "Failed to put some records in batch.";
m_metrics.IncrementRecordsPut(recordCount - failedPutCount);
m_metrics.IncrementRecordsFailed(failedPutCount);
}
else
{
log.Level = LogLevel.Debug;
log.Message = "Successfully put all records.";
m_metrics.IncrementRecordsPut(recordCount);
}
m_log.Log(log);
return(true);
} catch (Exception err) {
m_log.Error(err, "Failed to put record batch.");
m_metrics.IncrementPutRecordBatchErrors();
return(false);
}
}
protected override PutRecordBatchResponse SendRecords(List <Record> records, out bool successful)
{
var request = new PutRecordBatchRequest
{
DeliveryStreamName = _streamName,
Records = records
};
Logger.TraceFormat("Writing {0} records to firehose", records.Count);
var response = _kinesisFirehoseClient.PutRecordBatch(request);
successful = response.FailedPutCount == 0;
return(response);
}
protected override PutRecordBatchResponse SendRecords(List <Record> records, out bool successful)
{
var request = new PutRecordBatchRequest
{
DeliveryStreamName = _streamName,
Records = records
};
SelfLog.WriteLine("Writing {0} records to firehose", records.Count);
var putRecordBatchTask = _kinesisFirehoseClient.PutRecordBatchAsync(request);
successful = putRecordBatchTask.GetAwaiter().GetResult().FailedPutCount == 0;
return(putRecordBatchTask.Result);
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecordBatch operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecordBatch operation on AmazonKinesisFirehoseClient.</param>
/// <param name="callback">An Action delegate that is invoked when the operation completes.</param>
/// <param name="options">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/firehose-2015-08-04/PutRecordBatch">REST API Reference for PutRecordBatch Operation</seealso>
public virtual void PutRecordBatchAsync(PutRecordBatchRequest request, AmazonServiceCallback <PutRecordBatchRequest, PutRecordBatchResponse> callback, AsyncOptions options = null)
{
options = options == null?new AsyncOptions():options;
var marshaller = PutRecordBatchRequestMarshaller.Instance;
var unmarshaller = PutRecordBatchResponseUnmarshaller.Instance;
Action <AmazonWebServiceRequest, AmazonWebServiceResponse, Exception, AsyncOptions> callbackHelper = null;
if (callback != null)
{
callbackHelper = (AmazonWebServiceRequest req, AmazonWebServiceResponse res, Exception ex, AsyncOptions ao) => {
AmazonServiceResult <PutRecordBatchRequest, PutRecordBatchResponse> responseObject
= new AmazonServiceResult <PutRecordBatchRequest, PutRecordBatchResponse>((PutRecordBatchRequest)req, (PutRecordBatchResponse)res, ex, ao.State);
callback(responseObject);
}
}
;
BeginInvoke <PutRecordBatchRequest>(request, marshaller, unmarshaller, options, callbackHelper);
}
private void sendBatchInputsAsync()
{
try
{
string firehoseDelivaeryStreamName = getDeliveryStream();
PutRecordBatchRequest putRecordBatchRequest = new PutRecordBatchRequest
{
DeliveryStreamName = firehoseDelivaeryStreamName,
Records = m_InputsToSend.Select(createRecord).ToList()
};
m_Client.PutRecordBatchAsync(putRecordBatchRequest);
}
catch (Exception ex)
{
m_logger.LogError(string.Format("Failed to send monitoring statistics\n{0}", LogUtils.GetExceptionDetails(ex)));
//m_logger.LogError("Error", "Failed to send monitoring statistics", ex.Message + "\n" + ex.StackTrace);
}
}
public async Task <bool> RecordAsync <T>(IEnumerable <T> events, string streamName)
{
if (!events.Any())
{
return(false);
}
var records = new List <Record>();
foreach (T obj in events)
{
string objAsJson = _serializer.Serialize(events);
byte[] objAsBytes = Encoding.UTF8.GetBytes(objAsJson + "\n");
using MemoryStream ms = new MemoryStream(objAsBytes);
Record record = new Record {
Data = ms
};
records.Add(record);
}
var request = new PutRecordBatchRequest
{
DeliveryStreamName = streamName,
Records = records
};
PutRecordBatchResponse response = await _client.PutRecordBatchAsync(request);
if (response.HttpStatusCode != HttpStatusCode.OK)
{
throw new System.Exception($"Error sending message. HttpStatusCode: {response.HttpStatusCode}");
}
return(true);
}
/// <summary>
/// Emit a batch of log events, running to completion asynchronously.
/// </summary>
/// <param name="events">The events to be logged to Kinesis Firehose</param>
protected override Task EmitBatchAsync(IEnumerable <LogEvent> events)
{
var request = new PutRecordBatchRequest
{
DeliveryStreamName = _state.Options.StreamName
};
foreach (var logEvent in events)
{
var json = new StringWriter();
_state.Formatter.Format(logEvent, json);
var bytes = Encoding.UTF8.GetBytes(json.ToString());
var entry = new Record
{
Data = new MemoryStream(bytes),
};
request.Records.Add(entry);
}
return(_state.KinesisFirehoseClient.PutRecordBatchAsync(request));
}
