public static KFM.Record ReadFirehoseRecord(this BinaryReader reader)
{
var record = new KFM.Record();
record.Data = reader.ReadMemoryStream();
return(record);
}
/// <summary>
/// Emit a batch of log events, running to completion asynchronously.
/// </summary>
/// <param name="events">The events to be logged to Kinesis Firehose</param>
protected override void EmitBatch(IEnumerable<LogEvent> events)
{
var request = new PutRecordBatchRequest
{
DeliveryStreamName = _state.Options.StreamName
};
foreach (var logEvent in events)
{
var json = new StringWriter();
_state.Formatter.Format(logEvent, json);
var bytes = Encoding.UTF8.GetBytes(json.ToString());
var entry = new Record
{
Data = new MemoryStream(bytes),
};
request.Records.Add(entry);
}
_state.KinesisFirehoseClient.PutRecordBatch(request);
}
/// <summary>
/// Writes a single data record into an Amazon Kinesis Firehose delivery stream. To write
/// multiple data records into a delivery stream, use <a>PutRecordBatch</a>. Applications
/// using these operations are referred to as producers.
///
///
/// <para>
/// By default, each delivery stream can take in up to 2,000 transactions per second,
/// 5,000 records per second, or 5 MB per second. Note that if you use <a>PutRecord</a>
/// and <a>PutRecordBatch</a>, the limits are an aggregate across these two operations
/// for each delivery stream. For more information about limits and how to request an
/// increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Firehose Limits</a>.
/// </para>
///
/// <para>
/// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>.
/// The data record consists of a data blob that can be up to 1,000 KB in size, and any
/// kind of data, for example, a segment from a log file, geographic location data, web
/// site clickstream data, etc.
/// </para>
///
/// <para>
/// Firehose buffers records before delivering them to the destination. To disambiguate
/// the data blobs at the destination, a common solution is to use delimiters in the data,
/// such as a newline (<code>\n</code>) or some other character unique within the data.
/// This allows the consumer application(s) to parse individual data items when reading
/// the data from the destination.
/// </para>
///
/// <para>
/// The <a>PutRecord</a> operation returns a <b>RecordId</b>, which is a unique string
/// assigned to each record. Producer applications can use this ID for purposes such as
/// auditability and investigation.
/// </para>
///
/// <para>
/// If the <a>PutRecord</a> operation throws a <b>ServiceUnavailableException</b>, back
/// off and retry. If the exception persists, it is possible that the throughput limits
/// have been exceeded for the delivery stream.
/// </para>
///
/// <para>
/// Data records sent to Firehose are stored for 24 hours from the time they are added
/// to a delivery stream as it attempts to send the records to the destination. If the
/// destination is unreachable for more than 24 hours, the data is no longer available.
/// </para>
/// </summary>
/// <param name="deliveryStreamName">The name of the delivery stream.</param>
/// <param name="record">The record.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
///
/// <returns>The response from the PutRecord service method, as returned by KinesisFirehose.</returns>
/// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException">
/// The specified input parameter has an value that is not valid.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException">
/// The specified resource could not be found.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException">
/// The service is unavailable, back off and retry the operation. If you continue to see
/// the exception, throughput limits for the delivery stream may have been exceeded. For
/// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Firehose Limits</a>.
/// </exception>
public Task<PutRecordResponse> PutRecordAsync(string deliveryStreamName, Record record, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var request = new PutRecordRequest();
request.DeliveryStreamName = deliveryStreamName;
request.Record = record;
return PutRecordAsync(request, cancellationToken);
}
public static void WriteFirehoseRecord(this BinaryWriter writer, KFM.Record data)
{
writer.WriteMemoryStream(data.Data);
}
/// <summary>
/// Writes a single data record into an Amazon Kinesis Firehose delivery stream. To write
/// multiple data records into a delivery stream, use <a>PutRecordBatch</a>. Applications
/// using these operations are referred to as producers.
///
///
/// <para>
/// By default, each delivery stream can take in up to 2,000 transactions per second,
/// 5,000 records per second, or 5 MB per second. Note that if you use <a>PutRecord</a>
/// and <a>PutRecordBatch</a>, the limits are an aggregate across these two operations
/// for each delivery stream. For more information about limits and how to request an
/// increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Firehose Limits</a>.
/// </para>
///
/// <para>
/// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>.
/// The data record consists of a data blob that can be up to 1,000 KB in size, and any
/// kind of data, for example, a segment from a log file, geographic location data, web
/// site clickstream data, etc.
/// </para>
///
/// <para>
/// Firehose buffers records before delivering them to the destination. To disambiguate
/// the data blobs at the destination, a common solution is to use delimiters in the data,
/// such as a newline (<code>\n</code>) or some other character unique within the data.
/// This allows the consumer application(s) to parse individual data items when reading
/// the data from the destination.
/// </para>
///
/// <para>
/// The <a>PutRecord</a> operation returns a <b>RecordId</b>, which is a unique string
/// assigned to each record. Producer applications can use this ID for purposes such as
/// auditability and investigation.
/// </para>
///
/// <para>
/// If the <a>PutRecord</a> operation throws a <b>ServiceUnavailableException</b>, back
/// off and retry. If the exception persists, it is possible that the throughput limits
/// have been exceeded for the delivery stream.
/// </para>
///
/// <para>
/// Data records sent to Firehose are stored for 24 hours from the time they are added
/// to a delivery stream as it attempts to send the records to the destination. If the
/// destination is unreachable for more than 24 hours, the data is no longer available.
/// </para>
/// </summary>
/// <param name="deliveryStreamName">The name of the delivery stream.</param>
/// <param name="record">The record.</param>
///
/// <returns>The response from the PutRecord service method, as returned by KinesisFirehose.</returns>
/// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException">
/// The specified input parameter has an value that is not valid.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException">
/// The specified resource could not be found.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException">
/// The service is unavailable, back off and retry the operation. If you continue to see
/// the exception, throughput limits for the delivery stream may have been exceeded. For
/// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Firehose Limits</a>.
/// </exception>
public PutRecordResponse PutRecord(string deliveryStreamName, Record record)
{
var request = new PutRecordRequest();
request.DeliveryStreamName = deliveryStreamName;
request.Record = record;
return PutRecord(request);
}
/// <summary>
/// Writes a single data record into an Amazon Kinesis Firehose delivery stream. To write
/// multiple data records into a delivery stream, use <a>PutRecordBatch</a>. Applications
/// using these operations are referred to as producers.
///
///
/// <para>
/// By default, each delivery stream can take in up to 2,000 transactions per second,
/// 5,000 records per second, or 5 MB per second. Note that if you use <a>PutRecord</a>
/// and <a>PutRecordBatch</a>, the limits are an aggregate across these two operations
/// for each delivery stream. For more information about limits and how to request an
/// increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Firehose Limits</a>.
/// </para>
///
/// <para>
/// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>.
/// The data record consists of a data blob that can be up to 1,000 KB in size, and any
/// kind of data, for example, a segment from a log file, geographic location data, web
/// site clickstream data, etc.
/// </para>
///
/// <para>
/// Firehose buffers records before delivering them to the destination. To disambiguate
/// the data blobs at the destination, a common solution is to use delimiters in the data,
/// such as a newline (<code>\n</code>) or some other character unique within the data.
/// This allows the consumer application(s) to parse individual data items when reading
/// the data from the destination.
/// </para>
///
/// <para>
/// The <a>PutRecord</a> operation returns a <b>RecordId</b>, which is a unique string
/// assigned to each record. Producer applications can use this ID for purposes such as
/// auditability and investigation.
/// </para>
///
/// <para>
/// If the <a>PutRecord</a> operation throws a <b>ServiceUnavailableException</b>, back
/// off and retry. If the exception persists, it is possible that the throughput limits
/// have been exceeded for the delivery stream.
/// </para>
///
/// <para>
/// Data records sent to Firehose are stored for 24 hours from the time they are added
/// to a delivery stream as it attempts to send the records to the destination. If the
/// destination is unreachable for more than 24 hours, the data is no longer available.
/// </para>
/// </summary>
/// <param name="deliveryStreamName">The name of the delivery stream.</param>
/// <param name="record">The record.</param>
/// <param name="callback">An Action delegate that is invoked when the operation completes.</param>
/// <param name="options">
/// A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.
/// </param>
///
/// <returns>The response from the PutRecord service method, as returned by KinesisFirehose.</returns>
/// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException">
/// The specified input parameter has an value that is not valid.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException">
/// The specified resource could not be found.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException">
/// The service is unavailable, back off and retry the operation. If you continue to see
/// the exception, throughput limits for the delivery stream may have been exceeded. For
/// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Firehose Limits</a>.
/// </exception>
public void PutRecordAsync(string deliveryStreamName, Record record, AmazonServiceCallback<PutRecordRequest, PutRecordResponse> callback, AsyncOptions options = null)
{
var request = new PutRecordRequest();
request.DeliveryStreamName = deliveryStreamName;
request.Record = record;
PutRecordAsync(request, callback, options);
}
