public object Execute(ExecutorContext context) { var cmdletContext = context as CmdletContext; // create request var request = new Amazon.KinesisFirehose.Model.PutRecordBatchRequest(); if (cmdletContext.DeliveryStreamName != null) { request.DeliveryStreamName = cmdletContext.DeliveryStreamName; } if (cmdletContext.Record != null) { request.Records = cmdletContext.Record; } CmdletOutput output; // issue call var client = Client ?? CreateClient(_CurrentCredentials, _RegionEndpoint); try { var response = CallAWSServiceOperation(client, request); object pipelineOutput = null; pipelineOutput = cmdletContext.Select(response, this); output = new CmdletOutput { PipelineOutput = pipelineOutput, ServiceResponse = response }; } catch (Exception e) { output = new CmdletOutput { ErrorResponse = e }; } return(output); }
/// <summary> /// Emit a batch of log events, running to completion asynchronously. /// </summary> /// <param name="events">The events to be logged to Kinesis Firehose</param> protected override void EmitBatch(IEnumerable<LogEvent> events) { var request = new PutRecordBatchRequest { DeliveryStreamName = _state.Options.StreamName }; foreach (var logEvent in events) { var json = new StringWriter(); _state.Formatter.Format(logEvent, json); var bytes = Encoding.UTF8.GetBytes(json.ToString()); var entry = new Record { Data = new MemoryStream(bytes), }; request.Records.Add(entry); } _state.KinesisFirehoseClient.PutRecordBatch(request); }
/// <summary> /// Initiates the asynchronous execution of the PutRecordBatch operation. /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the PutRecordBatch operation.</param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// <returns>The task object representing the asynchronous operation.</returns> public Task<PutRecordBatchResponse> PutRecordBatchAsync(PutRecordBatchRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken)) { var marshaller = new PutRecordBatchRequestMarshaller(); var unmarshaller = PutRecordBatchResponseUnmarshaller.Instance; return InvokeAsync<PutRecordBatchRequest,PutRecordBatchResponse>(request, marshaller, unmarshaller, cancellationToken); }
/// <summary> /// Writes multiple data records into a delivery stream in a single call, which can achieve /// higher throughput per producer than when writing single records. To write single data /// records into a delivery stream, use <a>PutRecord</a>. Applications using these operations /// are referred to as producers. /// /// /// <para> /// Each <a>PutRecordBatch</a> request supports up to 500 records. Each record in the /// request can be as large as 1,000 KB (before 64-bit encoding), up to a limit of 4 MB /// for the entire request. By default, each delivery stream can take in up to 2,000 transactions /// per second, 5,000 records per second, or 5 MB per second. Note that if you use <a>PutRecord</a> /// and <a>PutRecordBatch</a>, the limits are an aggregate across these two operations /// for each delivery stream. For more information about limits and how to request an /// increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon /// Kinesis Firehose Limits</a>. /// </para> /// /// <para> /// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>. /// The data record consists of a data blob that can be up to 1,000 KB in size, and any /// kind of data, for example, a segment from a log file, geographic location data, web /// site clickstream data, and so on. /// </para> /// /// <para> /// Amazon Kinesis Firehose buffers records before delivering them to the destination. /// To disambiguate the data blobs at the destination, a common solution is to use delimiters /// in the data, such as a newline (<code>\n</code>) or some other character unique within /// the data. This allows the consumer application(s) to parse individual data items when /// reading the data from the destination. /// </para> /// /// <para> /// The <a>PutRecordBatch</a> response includes a count of any failed records, <code>FailedPutCount</code>, /// and an array of responses, <code>RequestResponses</code>. The <code>FailedPutCount</code> /// value is a count of records that failed. Each entry in the <code>RequestResponses</code> /// array gives additional information of the processed record. Each entry in <code>RequestResponses</code> /// directly correlates with a record in the request array using the same ordering, from /// the top to the bottom of the request and response. <code>RequestResponses</code> always /// includes the same number of records as the request array. <code>RequestResponses</code> /// both successfully and unsuccessfully processed records. Amazon Kinesis Firehose attempts /// to process all records in each <a>PutRecordBatch</a> request. A single record failure /// does not stop the processing of subsequent records. /// </para> /// /// <para> /// A successfully processed record includes a <code>RecordId</code> value, which is a /// unique value identified for the record. An unsuccessfully processed record includes /// <code>ErrorCode</code> and <code>ErrorMessage</code> values. <code>ErrorCode</code> /// reflects the type of error and is one of the following values: <code>ServiceUnavailable</code> /// or <code>InternalFailure</code>. <code>ErrorMessage</code> provides more detailed /// information about the error. /// </para> /// /// <para> /// If <code>FailedPutCount</code> is greater than 0 (zero), retry the request. A retry /// of the entire batch of records is possible; however, we strongly recommend that you /// inspect the entire response and resend only those records that failed processing. /// This minimizes duplicate records and also reduces the total bytes sent (and corresponding /// charges). /// </para> /// /// <para> /// If the <a>PutRecordBatch</a> operation throws a <code>ServiceUnavailableException</code>, /// back off and retry. If the exception persists, it is possible that the throughput /// limits have been exceeded for the delivery stream. /// </para> /// /// <para> /// Data records sent to Amazon Kinesis Firehose are stored for 24 hours from the time /// they are added to a delivery stream as it attempts to send the records to the destination. /// If the destination is unreachable for more than 24 hours, the data is no longer available. /// </para> /// </summary> /// <param name="request">Container for the necessary parameters to execute the PutRecordBatch service method.</param> /// /// <returns>The response from the PutRecordBatch service method, as returned by KinesisFirehose.</returns> /// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException"> /// The specified input parameter has an value that is not valid. /// </exception> /// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException"> /// The specified resource could not be found. /// </exception> /// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException"> /// The service is unavailable, back off and retry the operation. If you continue to see /// the exception, throughput limits for the delivery stream may have been exceeded. For /// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon /// Kinesis Firehose Limits</a>. /// </exception> public PutRecordBatchResponse PutRecordBatch(PutRecordBatchRequest request) { var marshaller = new PutRecordBatchRequestMarshaller(); var unmarshaller = PutRecordBatchResponseUnmarshaller.Instance; return Invoke<PutRecordBatchRequest,PutRecordBatchResponse>(request, marshaller, unmarshaller); }
/// <summary> /// Writes multiple data records into a delivery stream in a single call, which can achieve /// higher throughput per producer than when writing single records. To write single data /// records into a delivery stream, use <a>PutRecord</a>. Applications using these operations /// are referred to as producers. /// /// /// <para> /// By default, each delivery stream can take in up to 2,000 transactions per second, /// 5,000 records per second, or 5 MB per second. Note that if you use <a>PutRecord</a> /// and <a>PutRecordBatch</a>, the limits are an aggregate across these two operations /// for each delivery stream. For more information about limits, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon /// Kinesis Firehose Limits</a>. /// </para> /// /// <para> /// Each <a>PutRecordBatch</a> request supports up to 500 records. Each record in the /// request can be as large as 1,000 KB (before 64-bit encoding), up to a limit of 4 MB /// for the entire request. These limits cannot be changed. /// </para> /// /// <para> /// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>. /// The data record consists of a data blob that can be up to 1,000 KB in size, and any /// kind of data, for example, a segment from a log file, geographic location data, web /// site clickstream data, and so on. /// </para> /// /// <para> /// Firehose buffers records before delivering them to the destination. To disambiguate /// the data blobs at the destination, a common solution is to use delimiters in the data, /// such as a newline (<code>\n</code>) or some other character unique within the data. /// This allows the consumer application(s) to parse individual data items when reading /// the data from the destination. /// </para> /// /// <para> /// The <a>PutRecordBatch</a> response includes a count of failed records, <b>FailedPutCount</b>, /// and an array of responses, <b>RequestResponses</b>. Each entry in the <b>RequestResponses</b> /// array provides additional information about the processed record, and directly correlates /// with a record in the request array using the same ordering, from the top to the bottom. /// The response array always includes the same number of records as the request array. /// <b>RequestResponses</b> includes both successfully and unsuccessfully processed records. /// Firehose attempts to process all records in each <a>PutRecordBatch</a> request. A /// single record failure does not stop the processing of subsequent records. /// </para> /// /// <para> /// A successfully processed record includes a <b>RecordId</b> value, which is unique /// for the record. An unsuccessfully processed record includes <b>ErrorCode</b> and <b>ErrorMessage</b> /// values. <b>ErrorCode</b> reflects the type of error, and is one of the following values: /// <code>ServiceUnavailable</code> or <code>InternalFailure</code>. <b>ErrorMessage</b> /// provides more detailed information about the error. /// </para> /// /// <para> /// If there is an internal server error or a timeout, the write might have completed /// or it might have failed. If <b>FailedPutCount</b> is greater than 0, retry the request, /// resending only those records that might have failed processing. This minimizes the /// possible duplicate records and also reduces the total bytes sent (and corresponding /// charges). We recommend that you handle any duplicates at the destination. /// </para> /// /// <para> /// If <a>PutRecordBatch</a> throws <b>ServiceUnavailableException</b>, back off and retry. /// If the exception persists, it is possible that the throughput limits have been exceeded /// for the delivery stream. /// </para> /// /// <para> /// Data records sent to Firehose are stored for 24 hours from the time they are added /// to a delivery stream as it attempts to send the records to the destination. If the /// destination is unreachable for more than 24 hours, the data is no longer available. /// </para> /// </summary> /// <param name="deliveryStreamName">The name of the delivery stream.</param> /// <param name="records">One or more records.</param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// /// <returns>The response from the PutRecordBatch service method, as returned by KinesisFirehose.</returns> /// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException"> /// The specified input parameter has an value that is not valid. /// </exception> /// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException"> /// The specified resource could not be found. /// </exception> /// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException"> /// The service is unavailable, back off and retry the operation. If you continue to see /// the exception, throughput limits for the delivery stream may have been exceeded. For /// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon /// Kinesis Firehose Limits</a>. /// </exception> public Task<PutRecordBatchResponse> PutRecordBatchAsync(string deliveryStreamName, List<Record> records, System.Threading.CancellationToken cancellationToken = default(CancellationToken)) { var request = new PutRecordBatchRequest(); request.DeliveryStreamName = deliveryStreamName; request.Records = records; return PutRecordBatchAsync(request, cancellationToken); }
/// <summary> /// Initiates the asynchronous execution of the PutRecordBatch operation. /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the PutRecordBatch operation on AmazonKinesisFirehoseClient.</param> /// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param> /// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property.</param> /// /// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndPutRecordBatch /// operation.</returns> public IAsyncResult BeginPutRecordBatch(PutRecordBatchRequest request, AsyncCallback callback, object state) { var marshaller = new PutRecordBatchRequestMarshaller(); var unmarshaller = PutRecordBatchResponseUnmarshaller.Instance; return BeginInvoke<PutRecordBatchRequest>(request, marshaller, unmarshaller, callback, state); }
/// <summary> /// Writes multiple data records into a delivery stream in a single call, which can achieve /// higher throughput per producer than when writing single records. To write single data /// records into a delivery stream, use <a>PutRecord</a>. Applications using these operations /// are referred to as producers. /// /// /// <para> /// By default, each delivery stream can take in up to 2,000 transactions per second, /// 5,000 records per second, or 5 MB per second. Note that if you use <a>PutRecord</a> /// and <a>PutRecordBatch</a>, the limits are an aggregate across these two operations /// for each delivery stream. For more information about limits, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon /// Kinesis Firehose Limits</a>. /// </para> /// /// <para> /// Each <a>PutRecordBatch</a> request supports up to 500 records. Each record in the /// request can be as large as 1,000 KB (before 64-bit encoding), up to a limit of 4 MB /// for the entire request. These limits cannot be changed. /// </para> /// /// <para> /// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>. /// The data record consists of a data blob that can be up to 1,000 KB in size, and any /// kind of data, for example, a segment from a log file, geographic location data, web /// site clickstream data, and so on. /// </para> /// /// <para> /// Firehose buffers records before delivering them to the destination. To disambiguate /// the data blobs at the destination, a common solution is to use delimiters in the data, /// such as a newline (<code>\n</code>) or some other character unique within the data. /// This allows the consumer application(s) to parse individual data items when reading /// the data from the destination. /// </para> /// /// <para> /// The <a>PutRecordBatch</a> response includes a count of failed records, <b>FailedPutCount</b>, /// and an array of responses, <b>RequestResponses</b>. Each entry in the <b>RequestResponses</b> /// array provides additional information about the processed record, and directly correlates /// with a record in the request array using the same ordering, from the top to the bottom. /// The response array always includes the same number of records as the request array. /// <b>RequestResponses</b> includes both successfully and unsuccessfully processed records. /// Firehose attempts to process all records in each <a>PutRecordBatch</a> request. A /// single record failure does not stop the processing of subsequent records. /// </para> /// /// <para> /// A successfully processed record includes a <b>RecordId</b> value, which is unique /// for the record. An unsuccessfully processed record includes <b>ErrorCode</b> and <b>ErrorMessage</b> /// values. <b>ErrorCode</b> reflects the type of error, and is one of the following values: /// <code>ServiceUnavailable</code> or <code>InternalFailure</code>. <b>ErrorMessage</b> /// provides more detailed information about the error. /// </para> /// /// <para> /// If there is an internal server error or a timeout, the write might have completed /// or it might have failed. If <b>FailedPutCount</b> is greater than 0, retry the request, /// resending only those records that might have failed processing. This minimizes the /// possible duplicate records and also reduces the total bytes sent (and corresponding /// charges). We recommend that you handle any duplicates at the destination. /// </para> /// /// <para> /// If <a>PutRecordBatch</a> throws <b>ServiceUnavailableException</b>, back off and retry. /// If the exception persists, it is possible that the throughput limits have been exceeded /// for the delivery stream. /// </para> /// /// <para> /// Data records sent to Firehose are stored for 24 hours from the time they are added /// to a delivery stream as it attempts to send the records to the destination. If the /// destination is unreachable for more than 24 hours, the data is no longer available. /// </para> /// </summary> /// <param name="deliveryStreamName">The name of the delivery stream.</param> /// <param name="records">One or more records.</param> /// /// <returns>The response from the PutRecordBatch service method, as returned by KinesisFirehose.</returns> /// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException"> /// The specified input parameter has an value that is not valid. /// </exception> /// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException"> /// The specified resource could not be found. /// </exception> /// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException"> /// The service is unavailable, back off and retry the operation. If you continue to see /// the exception, throughput limits for the delivery stream may have been exceeded. For /// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon /// Kinesis Firehose Limits</a>. /// </exception> public PutRecordBatchResponse PutRecordBatch(string deliveryStreamName, List<Record> records) { var request = new PutRecordBatchRequest(); request.DeliveryStreamName = deliveryStreamName; request.Records = records; return PutRecordBatch(request); }
private Amazon.KinesisFirehose.Model.PutRecordBatchResponse CallAWSServiceOperation(IAmazonKinesisFirehose client, Amazon.KinesisFirehose.Model.PutRecordBatchRequest request) { Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Kinesis Firehose", "PutRecordBatch"); try { #if DESKTOP return(client.PutRecordBatch(request)); #elif CORECLR return(client.PutRecordBatchAsync(request).GetAwaiter().GetResult()); #else #error "Unknown build edition" #endif } catch (AmazonServiceException exc) { var webException = exc.InnerException as System.Net.WebException; if (webException != null) { throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException); } throw; } }
/// <summary> /// Initiates the asynchronous execution of the PutRecordBatch operation. /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the PutRecordBatch operation on AmazonKinesisFirehoseClient.</param> /// <param name="callback">An Action delegate that is invoked when the operation completes.</param> /// <param name="options">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property.</param> public void PutRecordBatchAsync(PutRecordBatchRequest request, AmazonServiceCallback<PutRecordBatchRequest, PutRecordBatchResponse> callback, AsyncOptions options = null) { options = options == null?new AsyncOptions():options; var marshaller = new PutRecordBatchRequestMarshaller(); var unmarshaller = PutRecordBatchResponseUnmarshaller.Instance; Action<AmazonWebServiceRequest, AmazonWebServiceResponse, Exception, AsyncOptions> callbackHelper = null; if(callback !=null ) callbackHelper = (AmazonWebServiceRequest req, AmazonWebServiceResponse res, Exception ex, AsyncOptions ao) => { AmazonServiceResult<PutRecordBatchRequest,PutRecordBatchResponse> responseObject = new AmazonServiceResult<PutRecordBatchRequest,PutRecordBatchResponse>((PutRecordBatchRequest)req, (PutRecordBatchResponse)res, ex , ao.State); callback(responseObject); }; BeginInvoke<PutRecordBatchRequest>(request, marshaller, unmarshaller, options, callbackHelper); }
/// <summary> /// Writes multiple data records into a delivery stream in a single call, which can achieve /// higher throughput per producer than when writing single records. To write single data /// records into a delivery stream, use <a>PutRecord</a>. Applications using these operations /// are referred to as producers. /// /// /// <para> /// By default, each delivery stream can take in up to 2,000 transactions per second, /// 5,000 records per second, or 5 MB per second. Note that if you use <a>PutRecord</a> /// and <a>PutRecordBatch</a>, the limits are an aggregate across these two operations /// for each delivery stream. For more information about limits, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon /// Kinesis Firehose Limits</a>. /// </para> /// /// <para> /// Each <a>PutRecordBatch</a> request supports up to 500 records. Each record in the /// request can be as large as 1,000 KB (before 64-bit encoding), up to a limit of 4 MB /// for the entire request. These limits cannot be changed. /// </para> /// /// <para> /// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>. /// The data record consists of a data blob that can be up to 1,000 KB in size, and any /// kind of data, for example, a segment from a log file, geographic location data, web /// site clickstream data, and so on. /// </para> /// /// <para> /// Firehose buffers records before delivering them to the destination. To disambiguate /// the data blobs at the destination, a common solution is to use delimiters in the data, /// such as a newline (<code>\n</code>) or some other character unique within the data. /// This allows the consumer application(s) to parse individual data items when reading /// the data from the destination. /// </para> /// /// <para> /// The <a>PutRecordBatch</a> response includes a count of failed records, <b>FailedPutCount</b>, /// and an array of responses, <b>RequestResponses</b>. Each entry in the <b>RequestResponses</b> /// array provides additional information about the processed record, and directly correlates /// with a record in the request array using the same ordering, from the top to the bottom. /// The response array always includes the same number of records as the request array. /// <b>RequestResponses</b> includes both successfully and unsuccessfully processed records. /// Firehose attempts to process all records in each <a>PutRecordBatch</a> request. A /// single record failure does not stop the processing of subsequent records. /// </para> /// /// <para> /// A successfully processed record includes a <b>RecordId</b> value, which is unique /// for the record. An unsuccessfully processed record includes <b>ErrorCode</b> and <b>ErrorMessage</b> /// values. <b>ErrorCode</b> reflects the type of error, and is one of the following values: /// <code>ServiceUnavailable</code> or <code>InternalFailure</code>. <b>ErrorMessage</b> /// provides more detailed information about the error. /// </para> /// /// <para> /// If there is an internal server error or a timeout, the write might have completed /// or it might have failed. If <b>FailedPutCount</b> is greater than 0, retry the request, /// resending only those records that might have failed processing. This minimizes the /// possible duplicate records and also reduces the total bytes sent (and corresponding /// charges). We recommend that you handle any duplicates at the destination. /// </para> /// /// <para> /// If <a>PutRecordBatch</a> throws <b>ServiceUnavailableException</b>, back off and retry. /// If the exception persists, it is possible that the throughput limits have been exceeded /// for the delivery stream. /// </para> /// /// <para> /// Data records sent to Firehose are stored for 24 hours from the time they are added /// to a delivery stream as it attempts to send the records to the destination. If the /// destination is unreachable for more than 24 hours, the data is no longer available. /// </para> /// </summary> /// <param name="deliveryStreamName">The name of the delivery stream.</param> /// <param name="records">One or more records.</param> /// <param name="callback">An Action delegate that is invoked when the operation completes.</param> /// <param name="options"> /// A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property. /// </param> /// /// <returns>The response from the PutRecordBatch service method, as returned by KinesisFirehose.</returns> /// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException"> /// The specified input parameter has an value that is not valid. /// </exception> /// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException"> /// The specified resource could not be found. /// </exception> /// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException"> /// The service is unavailable, back off and retry the operation. If you continue to see /// the exception, throughput limits for the delivery stream may have been exceeded. For /// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon /// Kinesis Firehose Limits</a>. /// </exception> public void PutRecordBatchAsync(string deliveryStreamName, List<Record> records, AmazonServiceCallback<PutRecordBatchRequest, PutRecordBatchResponse> callback, AsyncOptions options = null) { var request = new PutRecordBatchRequest(); request.DeliveryStreamName = deliveryStreamName; request.Records = records; PutRecordBatchAsync(request, callback, options); }