public void Post([FromBody] dynamic value) { string dataAsJson = JsonConvert.SerializeObject(value); byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson); using (MemoryStream memoryStream = new MemoryStream(dataAsBytes)) { try { AmazonKinesisConfig config = new AmazonKinesisConfig(); config.RegionEndpoint = Amazon.RegionEndpoint.USEast1; AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config); String kinesisStreamName = "click-stream"; PutRecordRequest requestRecord = new PutRecordRequest(); requestRecord.StreamName = kinesisStreamName; requestRecord.PartitionKey = "temp"; requestRecord.Data = memoryStream; kinesisClient.PutRecordAsync(requestRecord); //Console.WriteLine("Successfully sent record {0} to Kinesis. Sequence number: {1}", wt.Url, responseRecord.SequenceNumber); } catch (Exception ex) { Console.WriteLine("Failed to send record to Kinesis. Exception: {0}", ex.Message); } } }
//Just use for testing to put fake data into stream public void PutRecordsInStream() { try { var detail = new MyDetail() { SubscriberId = 1234567, FirstName = "test", LastName = "record" }; var myDetails = JsonConvert.SerializeObject(detail); var putRecordRequest = new PutRecordRequest() { Data = Utility.GenerateStreamFromString(myDetails), StreamName = _streamName, PartitionKey = "partitionKey-" + detail.FirstName }; var response = _amazonKinesisClient.PutRecord(putRecordRequest); } catch (Exception e) { Console.WriteLine(e); throw; } }
private static void PublishDeviceDataToKinesis(List <DeviceData> dataList) { // note: this constructor relies on you having set up a credential profile // named 'default', or have set credentials in environment variables // AWS_ACCESS_KEY_ID & AWS_SECRET_ACCESS_KEY, or have an application settings // file. See https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/net-dg-config-creds.html // for more details and other constructor options. var kinesisClient = new AmazonKinesisClient(_regionEndpoint); foreach (DeviceData data in dataList) { var dataAsJson = JsonConvert.SerializeObject(data); var dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson); using (var memoryStream = new MemoryStream(dataAsBytes)) { try { var requestRecord = new PutRecordRequest { StreamName = _kinesisStreamName, PartitionKey = data.DeviceId, Data = memoryStream }; var responseRecord = kinesisClient.PutRecordAsync(requestRecord).Result; Console.WriteLine($"Successfully published. Record:{data.DeviceId},{data.Humidity},{data.Temperature} Seq:{responseRecord.SequenceNumber}"); } catch (Exception ex) { Console.WriteLine($"Failed to publish. Exception: {ex.Message}"); } } } }
public async Task SendEvent(IHiarcEvent theEvent) { try { var accessKeyId = _kinesisSettings.AccessKeyId; var secretAccessKey = _kinesisSettings.SecretAccessKey; var streamName = _kinesisSettings.Stream; var partitionKey = theEvent.Event; var kinesisClient = new AmazonKinesisClient(accessKeyId, secretAccessKey, _region); var serializedEvent = JsonSerializer.Serialize(theEvent); using var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(theEvent.Event)); var requestRecord = new PutRecordRequest() { StreamName = streamName, PartitionKey = partitionKey, Data = memoryStream }; var responseRecord = await kinesisClient.PutRecordAsync(requestRecord); _logger.LogDebug($"Successfully sent event '{theEvent.Event}' to '{this.Name}'. Payload: {serializedEvent}"); } catch (Exception ex) { _logger.LogError($"Failed to send event '{theEvent.Event}' to '{this.Name}'. Exception: {ex.Message}"); } }
private static string PushData(string name, string userId) { try { var kinesis = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1); var mo = new { Name = name, UserId = userId }; var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(mo)); var ms = new MemoryStream(bytes); Random rnd = new Random(100); var request = new PutRecordRequest { StreamName = "your-kinesis-name", PartitionKey = "first-my-partion", Data = ms }; var kinesisResponse = kinesis.PutRecord(request); } catch (Exception ex) { return(ex.Message); } return("OK"); }
internal virtual PutRecordResponse PutRecord(PutRecordRequest request) { var marshaller = new PutRecordRequestMarshaller(); var unmarshaller = PutRecordResponseUnmarshaller.Instance; return(Invoke <PutRecordRequest, PutRecordResponse>(request, marshaller, unmarshaller)); }
/// /// Write to Firehose a record with the starting page and the page being requested. /// /// /// private void RecordRequest(Object source, EventArgs e) { // Create HttpApplication and HttpContext objects to access // request and response properties. HttpApplication application = (HttpApplication)source; HttpContext context = application.Context; string startingRequest = string.Empty; if (context.Request.UrlReferrer != null) { startingRequest = context.Request.UrlReferrer.PathAndQuery; } var record = new MemoryStream(UTF8Encoding.UTF8.GetBytes(string.Format("{0}t{1}n", startingRequest, context.Request.Path))); var request = new PutRecordRequest { DeliveryStreamName = this._deliveryStreamName, Record = new Record { Data = record } }; this._client.PutRecordAsync(request); }
public async Task <bool> RecordAsync <T>(T @event, string streamName) { string objAsJson = _serializer.Serialize(@event); byte[] objAsBytes = Encoding.UTF8.GetBytes(objAsJson + "\n"); using (var ms = new MemoryStream(objAsBytes)) { var record = new Record { Data = ms }; var request = new PutRecordRequest { DeliveryStreamName = streamName, Record = record }; PutRecordResponse response = await _client.PutRecordAsync(request); if (response.HttpStatusCode != HttpStatusCode.OK) { throw new System.Exception($"Error sending message. HttpStatusCode: {response.HttpStatusCode}"); } } return(true); }
/// <summary> /// Initiates the asynchronous execution of the PutRecord operation. /// <seealso cref="Amazon.Kinesis.IAmazonKinesis.PutRecord"/> /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the PutRecord operation.</param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// <returns>The task object representing the asynchronous operation.</returns> public Task <PutRecordResponse> PutRecordAsync(PutRecordRequest request, CancellationToken cancellationToken = default(CancellationToken)) { var marshaller = new PutRecordRequestMarshaller(); var unmarshaller = PutRecordResponseUnmarshaller.GetInstance(); return(Invoke <IRequest, PutRecordRequest, PutRecordResponse>(request, marshaller, unmarshaller, signer, cancellationToken)); }
private static void PublishDeviceDataToKinesis(List <DeviceData> dataList) { AmazonKinesisConfig config = new AmazonKinesisConfig(); config.RegionEndpoint = _regionEndpoint; AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config); foreach (DeviceData data in dataList) { string dataAsJson = JsonConvert.SerializeObject(data); byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson); using (MemoryStream memoryStream = new MemoryStream(dataAsBytes)) { try { PutRecordRequest requestRecord = new PutRecordRequest(); requestRecord.StreamName = _kinesisStreamName; requestRecord.PartitionKey = data.DeviceId; requestRecord.Data = memoryStream; PutRecordResponse responseRecord = kinesisClient.PutRecord(requestRecord); Console.WriteLine("Successfully published. Record:{0},{1},{2} Seq:{3}", data.DeviceId, data.Humidity, data.Temperature, responseRecord.SequenceNumber); } catch (Exception ex) { Console.WriteLine("Failed to publish. Exception: {0}", ex.Message); } } } }
private static void Main() { Console.WriteLine("Weather Stations Event Streamer" + Environment.NewLine); AmazonKinesisClient kinesisClient = null; try { //load config file var configuration = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json") .Build(); var kinesisStreamName = configuration["kinesis-stream"]; int timeDelay = Convert.ToInt16(configuration["time-delay"]); kinesisClient = new AmazonKinesisClient(region: Amazon.RegionEndpoint.USEast1); // Load mock json data from local file then convert to json var weatherEvents = JsonConvert.DeserializeObject <IList <Event> >(File.ReadAllText("weather-data.json")); foreach (var weatherEvent in weatherEvents) { // Send each weather item as an event to kinesis string dataAsJson = JsonConvert.SerializeObject(weatherEvent); using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(dataAsJson))) { var requestRecord = new PutRecordRequest { StreamName = kinesisStreamName, PartitionKey = "weather", Data = memoryStream }; var response = kinesisClient.PutRecordAsync(requestRecord).Result; if ((int)response.HttpStatusCode == 200) { Console.WriteLine($"Successfully sent record to Kinesis. Sequence number: {response.SequenceNumber}. Next event in {timeDelay}(s)"); } else { Console.Write($"Failed on: {dataAsJson}"); } } Thread.Sleep(timeDelay * 1000); } } catch (AmazonClientException amazonEx) { Console.WriteLine("Opps! Looks like you don't have an Amazon cli setup or incorrect Kinesis stream name. Update appsettings.json in this build directory. Check the exception below for more details."); Console.Write(amazonEx.GetBaseException().ToString()); Console.ReadLine(); throw; } catch (Exception ex) { Console.Write(ex.GetBaseException().ToString()); Console.ReadLine(); throw; } finally { kinesisClient?.Dispose(); } }
/// <summary> /// Sends sensor data to AWS Kinesis Firehose. /// </summary> /// <param name="temp">Temperature value</param> /// <param name="hum">Humidity value</param> /// <param name="bat">Battery level</param> private async void SendMessageToAWSFirehose(double temp, uint hum, uint bat) { // Check that the client is not null (might be if initialization failed) if (_client != null) { PutRecordRequest req = new PutRecordRequest(); req.DeliveryStreamName = AWS_DELIVERY_STREAM; // Create the message // Message format: ISO timestamp, temperature, humidity, battery String data = String.Format("{0},{1},{2},{3}\n", DateTime.UtcNow.ToString("u"), temp, hum, bat); try { var record = new Record { Data = new MemoryStream(UTF8Encoding.UTF8.GetBytes(data)) }; req.Record = record; await _client.PutRecordAsync(req); } catch (Exception e) { // Notify the user await this.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () => { statusText.Text = string.Format("Error sending data to cloud: {0}", e.ToString()); }); } } }
private async Task WriteToStream() { const string myStreamName = "myTestStream"; const int myStreamSize = 1; try { var createStreamRequest = new CreateStreamRequest(); createStreamRequest.StreamName = myStreamName; createStreamRequest.ShardCount = myStreamSize; var createStreamReq = createStreamRequest; var existingStreams = await kinesisClient.ListStreamsAsync(); if (!existingStreams.StreamNames.Contains(myStreamName)) { var CreateStreamResponse = await kinesisClient.CreateStreamAsync(createStreamReq); Console.WriteLine("Created Stream : " + myStreamName); } } catch (ResourceInUseException) { Console.Error.WriteLine("Producer is quitting without creating stream " + myStreamName + " to put records into as a stream of the same name already exists."); Environment.Exit(1); } await WaitForStreamToBecomeAvailableAsync(myStreamName); Console.Error.WriteLine("Putting records in stream : " + myStreamName); // Write 10 UTF-8 encoded records to the stream. for (int j = 0; j < 10; ++j) { byte[] dataAsBytes = Encoding.UTF8.GetBytes("testdata-" + j); using (MemoryStream memoryStream = new MemoryStream(dataAsBytes)) { try { PutRecordRequest requestRecord = new PutRecordRequest(); requestRecord.StreamName = myStreamName; requestRecord.PartitionKey = "url-response-times"; requestRecord.Data = memoryStream; PutRecordResponse responseRecord = await kinesisClient.PutRecordAsync(requestRecord); Console.WriteLine("Successfully sent record to Kinesis. Sequence number: {0}", responseRecord.SequenceNumber); } catch (Exception ex) { Console.WriteLine("Failed to send record to Kinesis. Exception: {0}", ex.Message); } } } Console.ReadLine(); }
/// <summary> /// Initiates the asynchronous execution of the PutRecord operation. /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the PutRecord operation on AmazonKinesisFirehoseClient.</param> /// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param> /// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property.</param> /// /// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndPutRecord /// operation.</returns> public IAsyncResult BeginPutRecord(PutRecordRequest request, AsyncCallback callback, object state) { var marshaller = new PutRecordRequestMarshaller(); var unmarshaller = PutRecordResponseUnmarshaller.Instance; return(BeginInvoke <PutRecordRequest>(request, marshaller, unmarshaller, callback, state)); }
/// <summary> /// Writes a single data record into an Amazon Kinesis Firehose delivery stream. To write /// multiple data records into a delivery stream, use <a>PutRecordBatch</a>. Applications /// using these operations are referred to as producers. /// /// /// <para> /// By default, each delivery stream can take in up to 2,000 transactions per second, /// 5,000 records per second, or 5 MB per second. Note that if you use <a>PutRecord</a> /// and <a>PutRecordBatch</a>, the limits are an aggregate across these two operations /// for each delivery stream. For more information about limits and how to request an /// increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon /// Kinesis Firehose Limits</a>. /// </para> /// /// <para> /// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>. /// The data record consists of a data blob that can be up to 1,000 KB in size, and any /// kind of data, for example, a segment from a log file, geographic location data, website /// clickstream data, and so on. /// </para> /// /// <para> /// Kinesis Firehose buffers records before delivering them to the destination. To disambiguate /// the data blobs at the destination, a common solution is to use delimiters in the data, /// such as a newline (<code>\n</code>) or some other character unique within the data. /// This allows the consumer application to parse individual data items when reading the /// data from the destination. /// </para> /// /// <para> /// The <a>PutRecord</a> operation returns a <b>RecordId</b>, which is a unique string /// assigned to each record. Producer applications can use this ID for purposes such as /// auditability and investigation. /// </para> /// /// <para> /// If the <a>PutRecord</a> operation throws a <b>ServiceUnavailableException</b>, back /// off and retry. If the exception persists, it is possible that the throughput limits /// have been exceeded for the delivery stream. /// </para> /// /// <para> /// Data records sent to Kinesis Firehose are stored for 24 hours from the time they are /// added to a delivery stream as it attempts to send the records to the destination. /// If the destination is unreachable for more than 24 hours, the data is no longer available. /// </para> /// </summary> /// <param name="deliveryStreamName">The name of the delivery stream.</param> /// <param name="record">The record.</param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// /// <returns>The response from the PutRecord service method, as returned by KinesisFirehose.</returns> /// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException"> /// The specified input parameter has a value that is not valid. /// </exception> /// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException"> /// The specified resource could not be found. /// </exception> /// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException"> /// The service is unavailable, back off and retry the operation. If you continue to see /// the exception, throughput limits for the delivery stream may have been exceeded. For /// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon /// Kinesis Firehose Limits</a>. /// </exception> /// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/firehose-2015-08-04/PutRecord">REST API Reference for PutRecord Operation</seealso> public virtual Task <PutRecordResponse> PutRecordAsync(string deliveryStreamName, Record record, System.Threading.CancellationToken cancellationToken = default(CancellationToken)) { var request = new PutRecordRequest(); request.DeliveryStreamName = deliveryStreamName; request.Record = record; return(PutRecordAsync(request, cancellationToken)); }
/// <summary> /// Initiates the asynchronous execution of the PutRecord operation. /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the PutRecord operation.</param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// <returns>The task object representing the asynchronous operation.</returns> /// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/firehose-2015-08-04/PutRecord">REST API Reference for PutRecord Operation</seealso> public virtual Task <PutRecordResponse> PutRecordAsync(PutRecordRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken)) { var marshaller = new PutRecordRequestMarshaller(); var unmarshaller = PutRecordResponseUnmarshaller.Instance; return(InvokeAsync <PutRecordRequest, PutRecordResponse>(request, marshaller, unmarshaller, cancellationToken)); }
/// <summary> /// Initiates the asynchronous execution of the PutRecord operation. /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the PutRecord operation on AmazonSageMakerFeatureStoreRuntimeClient.</param> /// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param> /// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property.</param> /// /// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndPutRecord /// operation.</returns> /// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/sagemaker-featurestore-runtime-2020-07-01/PutRecord">REST API Reference for PutRecord Operation</seealso> public virtual IAsyncResult BeginPutRecord(PutRecordRequest request, AsyncCallback callback, object state) { var options = new InvokeOptions(); options.RequestMarshaller = PutRecordRequestMarshaller.Instance; options.ResponseUnmarshaller = PutRecordResponseUnmarshaller.Instance; return BeginInvoke(request, options, callback, state); }
/// <summary> /// Used for data ingestion into the <code>FeatureStore</code>. The <code>PutRecord</code> /// API writes to both the <code>OnlineStore</code> and <code>OfflineStore</code>. If /// the record is the latest record for the <code>recordIdentifier</code>, the record /// is written to both the <code>OnlineStore</code> and <code>OfflineStore</code>. If /// the record is a historic record, it is written only to the <code>OfflineStore</code>. /// </summary> /// <param name="request">Container for the necessary parameters to execute the PutRecord service method.</param> /// /// <returns>The response from the PutRecord service method, as returned by SageMakerFeatureStoreRuntime.</returns> /// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.AccessForbiddenException"> /// You do not have permission to perform an action. /// </exception> /// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.InternalFailureException"> /// An internal failure occurred. Try your request again. If the problem persists, contact /// AWS customer support. /// </exception> /// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.ServiceUnavailableException"> /// The service is currently unavailable. /// </exception> /// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.ValidationErrorException"> /// There was an error validating your request. /// </exception> /// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/sagemaker-featurestore-runtime-2020-07-01/PutRecord">REST API Reference for PutRecord Operation</seealso> public virtual PutRecordResponse PutRecord(PutRecordRequest request) { var options = new InvokeOptions(); options.RequestMarshaller = PutRecordRequestMarshaller.Instance; options.ResponseUnmarshaller = PutRecordResponseUnmarshaller.Instance; return Invoke<PutRecordResponse>(request, options); }
/// <summary> /// Writes a single data record into an Amazon Kinesis Data Firehose delivery stream. /// To write multiple data records into a delivery stream, use <a>PutRecordBatch</a>. /// Applications using these operations are referred to as producers. /// /// /// <para> /// By default, each delivery stream can take in up to 2,000 transactions per second, /// 5,000 records per second, or 5 MB per second. If you use <a>PutRecord</a> and <a>PutRecordBatch</a>, /// the limits are an aggregate across these two operations for each delivery stream. /// For more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon /// Kinesis Data Firehose Limits</a>. /// </para> /// /// <para> /// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>. /// The data record consists of a data blob that can be up to 1,000 KB in size, and any /// kind of data. For example, it can be a segment from a log file, geographic location /// data, website clickstream data, and so on. /// </para> /// /// <para> /// Kinesis Data Firehose buffers records before delivering them to the destination. To /// disambiguate the data blobs at the destination, a common solution is to use delimiters /// in the data, such as a newline (<code>\n</code>) or some other character unique within /// the data. This allows the consumer application to parse individual data items when /// reading the data from the destination. /// </para> /// /// <para> /// The <code>PutRecord</code> operation returns a <code>RecordId</code>, which is a unique /// string assigned to each record. Producer applications can use this ID for purposes /// such as auditability and investigation. /// </para> /// /// <para> /// If the <code>PutRecord</code> operation throws a <code>ServiceUnavailableException</code>, /// back off and retry. If the exception persists, it is possible that the throughput /// limits have been exceeded for the delivery stream. /// </para> /// /// <para> /// Data records sent to Kinesis Data Firehose are stored for 24 hours from the time they /// are added to a delivery stream as it tries to send the records to the destination. /// If the destination is unreachable for more than 24 hours, the data is no longer available. /// </para> /// <important> /// <para> /// Don't concatenate two or more base64 strings to form the data fields of your records. /// Instead, concatenate the raw data, then perform base64 encoding. /// </para> /// </important> /// </summary> /// <param name="deliveryStreamName">The name of the delivery stream.</param> /// <param name="record">The record.</param> /// <param name="callback">An Action delegate that is invoked when the operation completes.</param> /// <param name="options"> /// A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property. /// </param> /// /// <returns>The response from the PutRecord service method, as returned by KinesisFirehose.</returns> /// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException"> /// The specified input parameter has a value that is not valid. /// </exception> /// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException"> /// The specified resource could not be found. /// </exception> /// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException"> /// The service is unavailable. Back off and retry the operation. If you continue to see /// the exception, throughput limits for the delivery stream may have been exceeded. For /// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon /// Kinesis Data Firehose Limits</a>. /// </exception> /// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/firehose-2015-08-04/PutRecord">REST API Reference for PutRecord Operation</seealso> public virtual void PutRecordAsync(string deliveryStreamName, Record record, AmazonServiceCallback <PutRecordRequest, PutRecordResponse> callback, AsyncOptions options = null) { var request = new PutRecordRequest(); request.DeliveryStreamName = deliveryStreamName; request.Record = record; PutRecordAsync(request, callback, options); }
/// <summary> /// This method verifies your credentials, creates a Kinesis stream, waits for the stream /// to become active, then puts 10 records in it, and (optionally) deletes the stream. /// </summary> public async static Task Main(string[] args) { const string myStreamName = "myTestStream"; const int myStreamSize = 1; try { var createStreamRequest = new CreateStreamRequest(); createStreamRequest.StreamName = myStreamName; createStreamRequest.ShardCount = myStreamSize; var createStreamReq = createStreamRequest; await kinesisClient.CreateStreamAsync(createStreamReq); Console.Error.WriteLine("Created Stream : " + myStreamName); } catch (ResourceInUseException) { Console.Error.WriteLine("Producer is quitting without creating stream " + myStreamName + " to put records into as a stream of the same name already exists."); Environment.Exit(1); } await WaitForStreamToBecomeAvailable(myStreamName); Console.Error.WriteLine("Putting records in stream : " + myStreamName); // Write 10 UTF-8 encoded records to the stream. for (int j = 0; j < 10; ++j) { PutRecordRequest requestRecord = new PutRecordRequest(); requestRecord.StreamName = myStreamName; requestRecord.Data = new MemoryStream(Encoding.UTF8.GetBytes("testData-" + j)); requestRecord.PartitionKey = "partitionKey-" + j; var putResult = await kinesisClient.PutRecordAsync(requestRecord); Console.Error.WriteLine( String.Format("Successfully putrecord {0}:\n\t partition key = {1,15}, shard ID = {2}", j, requestRecord.PartitionKey, putResult.ShardId)); } // Uncomment the following if you wish to delete the stream here. //Console.Error.WriteLine("Deleting stream : " + myStreamName); //DeleteStreamRequest deleteStreamReq = new DeleteStreamRequest(); //deleteStreamReq.StreamName = myStreamName; //try //{ // kinesisClient.DeleteStream(deleteStreamReq); // Console.Error.WriteLine("Stream is now being deleted : " + myStreamName); //} //catch (ResourceNotFoundException ex) // // Console.Error.WriteLine("Stream could not be found; " + ex); //} //catch (AmazonClientException ex) //{ // Console.Error.WriteLine("Error deleting stream; " + ex); //} }
/// <summary> /// Used for data ingestion into the <code>FeatureStore</code>. The <code>PutRecord</code> /// API writes to both the <code>OnlineStore</code> and <code>OfflineStore</code>. If /// the record is the latest record for the <code>recordIdentifier</code>, the record /// is written to both the <code>OnlineStore</code> and <code>OfflineStore</code>. If /// the record is a historic record, it is written only to the <code>OfflineStore</code>. /// </summary> /// <param name="request">Container for the necessary parameters to execute the PutRecord service method.</param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// /// <returns>The response from the PutRecord service method, as returned by SageMakerFeatureStoreRuntime.</returns> /// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.AccessForbiddenException"> /// You do not have permission to perform an action. /// </exception> /// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.InternalFailureException"> /// An internal failure occurred. Try your request again. If the problem persists, contact /// AWS customer support. /// </exception> /// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.ServiceUnavailableException"> /// The service is currently unavailable. /// </exception> /// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.ValidationErrorException"> /// There was an error validating your request. /// </exception> /// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/sagemaker-featurestore-runtime-2020-07-01/PutRecord">REST API Reference for PutRecord Operation</seealso> public virtual Task <PutRecordResponse> PutRecordAsync(PutRecordRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken)) { var options = new InvokeOptions(); options.RequestMarshaller = PutRecordRequestMarshaller.Instance; options.ResponseUnmarshaller = PutRecordResponseUnmarshaller.Instance; return(InvokeAsync <PutRecordResponse>(request, options, cancellationToken)); }
public void Send(Dictionary <string, object> logEvent) { var requestRecord = new PutRecordRequest { StreamName = StreamName, Data = new MemoryStream(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(logEvent))), PartitionKey = AppName }; Task.Run(() => _client.PutRecordAsync(requestRecord)); }
public async Task <PutRecordResponse> PutKinesisRecord(string data, string partitionKey) { var putRecordRequest = new PutRecordRequest { StreamName = _StreamName, PartitionKey = partitionKey, Data = new MemoryStream(Encoding.UTF8.GetBytes(data)) }; return(await _AmazonKinesisClient.PutRecordAsync(putRecordRequest)); }
public async Task SendMessageAsync(TMessage message) { PutRecordRequest requestRecord = new PutRecordRequest(); requestRecord.StreamName = streamName; requestRecord.Data = new MemoryStream(serializer.SerializeToArray(message)); requestRecord.PartitionKey = partitionKeyFunc(message); await kinesisClient.PutRecordAsync(requestRecord); }
public void LogRecord(Record record) { byte[] dataAsBytes = Encoding.UTF8.GetBytes(record.ToJson()); using (MemoryStream memoryStream = new MemoryStream(dataAsBytes)) { PutRecordRequest requestRecord = new PutRecordRequest(); requestRecord.StreamName = StreamName; requestRecord.PartitionKey = PartitionKey; requestRecord.Data = memoryStream; PutRecordResponse responseRecord = KinesisClient.PutRecord(requestRecord); } }
public async void Accept(CommandChunk chunk) { var bytes = _serializer.Serialize(chunk); var request = new PutRecordRequest { SequenceNumberForOrdering = chunk.EngineSequenceNumber.ToString(), StreamName = _streamName, Data = new MemoryStream(bytes), PartitionKey = chunk.Engine.ToString() //todo: ordering guarantees? }; var response = await _client.PutRecordAsync(request); var number = response.SequenceNumber; }
/// <summary> /// <para>This operation puts a data record into an Amazon Kinesis stream from a producer. This operation must be called to send data from the /// producer into the Amazon Kinesis stream for real-time ingestion and subsequent processing. The <c>PutRecord</c> operation requires the name /// of the stream that captures, stores, and transports the data; a partition key; and the data blob itself. The data blob could be a segment /// from a log file, geographic/location data, website clickstream data, or any other data type.</para> <para>The partition key is used to /// distribute data across shards. Amazon Kinesis segregates the data records that belong to a data stream into multiple shards, using the /// partition key associated with each data record to determine which shard a given data record belongs to. </para> <para>Partition keys are /// Unicode strings, with a maximum length limit of 256 bytes. An MD5 hash function is used to map partition keys to 128-bit integer values and /// to map associated data records to shards using the hash key ranges of the shards. You can override hashing the partition key to determine /// the shard by explicitly specifying a hash value using the <c>ExplicitHashKey</c> parameter. For more information, see the <a href="http://docs.aws.amazon.com/kinesis/latest/dev/">Amazon Kinesis Developer Guide</a> .</para> <para> <c>PutRecord</c> returns the shard /// ID of where the data record was placed and the sequence number that was assigned to the data record.</para> <para>Sequence numbers generally /// increase over time. To guarantee strictly increasing ordering, use the <c>SequenceNumberForOrdering</c> parameter. For more information, see /// the <a href="http://docs.aws.amazon.com/kinesis/latest/dev/">Amazon Kinesis Developer Guide</a> .</para> <para>If a <c>PutRecord</c> /// request cannot be processed because of insufficient provisioned throughput on the shard involved in the request, <c>PutRecord</c> throws /// <c>ProvisionedThroughputExceededException</c> . </para> <para>Data records are accessible for only 24 hours from the time that they are /// added to an Amazon Kinesis stream.</para> /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the PutRecord service method on AmazonKinesis.</param> /// /// <returns>The response from the PutRecord service method, as returned by AmazonKinesis.</returns> /// /// <exception cref="T:Amazon.Kinesis.Model.ProvisionedThroughputExceededException" /> /// <exception cref="T:Amazon.Kinesis.Model.InvalidArgumentException" /> /// <exception cref="T:Amazon.Kinesis.Model.ResourceNotFoundException" /> public PutRecordResponse PutRecord(PutRecordRequest request) { var task = PutRecordAsync(request); try { return(task.Result); } catch (AggregateException e) { ExceptionDispatchInfo.Capture(e.InnerException).Throw(); return(null); } }
public async Task RecordAsync(Activity record, string partitionKey) { var bytes = Serialize(record); using (var ms = new MemoryStream(bytes)) { var putRecordRequest = new PutRecordRequest { StreamName = _streamName, Data = ms, PartitionKey = partitionKey }; await _kinesis.PutRecordAsync(putRecordRequest); } }
public async void Handle(CommandChunk chunk) { var bytes = _serializer.Serialize(chunk); var request = new PutRecordRequest { SequenceNumberForOrdering = chunk.LocalSequenceNumber.ToString(), StreamName = _streamName, Data = new MemoryStream(bytes), PartitionKey = chunk.PartitionKey }; var response = await _client.PutRecordAsync(request); chunk.GlobalSequenceNumber = response.SequenceNumber; }
public async Task <string> PutAsync(Message message, string streamName) { var json = JsonConvert.SerializeObject(message); using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(json))) { var putRecordRequest = new PutRecordRequest { DeliveryStreamName = streamName, Record = new Record { Data = stream } }; var response = await _client.PutRecordAsync(putRecordRequest); return(response.RecordId); } }