public void Post([FromBody] dynamic value)
{
string dataAsJson = JsonConvert.SerializeObject(value);
byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson);
using (MemoryStream memoryStream = new MemoryStream(dataAsBytes))
{
try
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = Amazon.RegionEndpoint.USEast1;
AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
String kinesisStreamName = "click-stream";
PutRecordRequest requestRecord = new PutRecordRequest();
requestRecord.StreamName = kinesisStreamName;
requestRecord.PartitionKey = "temp";
requestRecord.Data = memoryStream;
kinesisClient.PutRecordAsync(requestRecord);
//Console.WriteLine("Successfully sent record {0} to Kinesis. Sequence number: {1}", wt.Url, responseRecord.SequenceNumber);
}
catch (Exception ex)
{
Console.WriteLine("Failed to send record to Kinesis. Exception: {0}", ex.Message);
}
}
}
//Just use for testing to put fake data into stream
public void PutRecordsInStream()
{
try
{
var detail = new MyDetail()
{
SubscriberId = 1234567,
FirstName = "test",
LastName = "record"
};
var myDetails = JsonConvert.SerializeObject(detail);
var putRecordRequest = new PutRecordRequest()
{
Data = Utility.GenerateStreamFromString(myDetails),
StreamName = _streamName,
PartitionKey = "partitionKey-" + detail.FirstName
};
var response = _amazonKinesisClient.PutRecord(putRecordRequest);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
private static void PublishDeviceDataToKinesis(List <DeviceData> dataList)
{
// note: this constructor relies on you having set up a credential profile
// named 'default', or have set credentials in environment variables
// AWS_ACCESS_KEY_ID & AWS_SECRET_ACCESS_KEY, or have an application settings
// file. See https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/net-dg-config-creds.html
// for more details and other constructor options.
var kinesisClient = new AmazonKinesisClient(_regionEndpoint);
foreach (DeviceData data in dataList)
{
var dataAsJson = JsonConvert.SerializeObject(data);
var dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson);
using (var memoryStream = new MemoryStream(dataAsBytes))
{
try
{
var requestRecord = new PutRecordRequest
{
StreamName = _kinesisStreamName,
PartitionKey = data.DeviceId,
Data = memoryStream
};
var responseRecord = kinesisClient.PutRecordAsync(requestRecord).Result;
Console.WriteLine($"Successfully published. Record:{data.DeviceId},{data.Humidity},{data.Temperature} Seq:{responseRecord.SequenceNumber}");
}
catch (Exception ex)
{
Console.WriteLine($"Failed to publish. Exception: {ex.Message}");
}
}
}
}
public async Task SendEvent(IHiarcEvent theEvent)
{
try
{
var accessKeyId = _kinesisSettings.AccessKeyId;
var secretAccessKey = _kinesisSettings.SecretAccessKey;
var streamName = _kinesisSettings.Stream;
var partitionKey = theEvent.Event;
var kinesisClient = new AmazonKinesisClient(accessKeyId, secretAccessKey, _region);
var serializedEvent = JsonSerializer.Serialize(theEvent);
using var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(theEvent.Event));
var requestRecord = new PutRecordRequest()
{
StreamName = streamName, PartitionKey = partitionKey, Data = memoryStream
};
var responseRecord = await kinesisClient.PutRecordAsync(requestRecord);
_logger.LogDebug($"Successfully sent event '{theEvent.Event}' to '{this.Name}'. Payload: {serializedEvent}");
}
catch (Exception ex)
{
_logger.LogError($"Failed to send event '{theEvent.Event}' to '{this.Name}'. Exception: {ex.Message}");
}
}
private static string PushData(string name, string userId)
{
try
{
var kinesis = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1);
var mo = new { Name = name, UserId = userId };
var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(mo));
var ms = new MemoryStream(bytes);
Random rnd = new Random(100);
var request = new PutRecordRequest
{
StreamName = "your-kinesis-name",
PartitionKey = "first-my-partion",
Data = ms
};
var kinesisResponse = kinesis.PutRecord(request);
}
catch (Exception ex)
{
return(ex.Message);
}
return("OK");
}
internal virtual PutRecordResponse PutRecord(PutRecordRequest request)
{
var marshaller = new PutRecordRequestMarshaller();
var unmarshaller = PutRecordResponseUnmarshaller.Instance;
return(Invoke <PutRecordRequest, PutRecordResponse>(request, marshaller, unmarshaller));
}
///
/// Write to Firehose a record with the starting page and the page being requested.
///
///
///
private void RecordRequest(Object source, EventArgs e)
{
// Create HttpApplication and HttpContext objects to access
// request and response properties.
HttpApplication application = (HttpApplication)source;
HttpContext context = application.Context;
string startingRequest = string.Empty;
if (context.Request.UrlReferrer != null)
{
startingRequest = context.Request.UrlReferrer.PathAndQuery;
}
var record = new MemoryStream(UTF8Encoding.UTF8.GetBytes(string.Format("{0}t{1}n",
startingRequest, context.Request.Path)));
var request = new PutRecordRequest
{
DeliveryStreamName = this._deliveryStreamName,
Record = new Record
{
Data = record
}
};
this._client.PutRecordAsync(request);
}
public async Task <bool> RecordAsync <T>(T @event, string streamName)
{
string objAsJson = _serializer.Serialize(@event);
byte[] objAsBytes = Encoding.UTF8.GetBytes(objAsJson + "\n");
using (var ms = new MemoryStream(objAsBytes))
{
var record = new Record {
Data = ms
};
var request = new PutRecordRequest
{
DeliveryStreamName = streamName,
Record = record
};
PutRecordResponse response = await _client.PutRecordAsync(request);
if (response.HttpStatusCode != HttpStatusCode.OK)
{
throw new System.Exception($"Error sending message. HttpStatusCode: {response.HttpStatusCode}");
}
}
return(true);
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecord operation.
/// <seealso cref="Amazon.Kinesis.IAmazonKinesis.PutRecord"/>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecord operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task <PutRecordResponse> PutRecordAsync(PutRecordRequest request, CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new PutRecordRequestMarshaller();
var unmarshaller = PutRecordResponseUnmarshaller.GetInstance();
return(Invoke <IRequest, PutRecordRequest, PutRecordResponse>(request, marshaller, unmarshaller, signer, cancellationToken));
}
private static void PublishDeviceDataToKinesis(List <DeviceData> dataList)
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = _regionEndpoint;
AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
foreach (DeviceData data in dataList)
{
string dataAsJson = JsonConvert.SerializeObject(data);
byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson);
using (MemoryStream memoryStream = new MemoryStream(dataAsBytes))
{
try
{
PutRecordRequest requestRecord = new PutRecordRequest();
requestRecord.StreamName = _kinesisStreamName;
requestRecord.PartitionKey = data.DeviceId;
requestRecord.Data = memoryStream;
PutRecordResponse responseRecord = kinesisClient.PutRecord(requestRecord);
Console.WriteLine("Successfully published. Record:{0},{1},{2} Seq:{3}",
data.DeviceId, data.Humidity, data.Temperature, responseRecord.SequenceNumber);
}
catch (Exception ex)
{
Console.WriteLine("Failed to publish. Exception: {0}", ex.Message);
}
}
}
}
private static void Main()
{
Console.WriteLine("Weather Stations Event Streamer" + Environment.NewLine);
AmazonKinesisClient kinesisClient = null;
try
{
//load config file
var configuration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json")
.Build();
var kinesisStreamName = configuration["kinesis-stream"];
int timeDelay = Convert.ToInt16(configuration["time-delay"]);
kinesisClient = new AmazonKinesisClient(region: Amazon.RegionEndpoint.USEast1);
// Load mock json data from local file then convert to json
var weatherEvents = JsonConvert.DeserializeObject <IList <Event> >(File.ReadAllText("weather-data.json"));
foreach (var weatherEvent in weatherEvents)
{
// Send each weather item as an event to kinesis
string dataAsJson = JsonConvert.SerializeObject(weatherEvent);
using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(dataAsJson)))
{
var requestRecord = new PutRecordRequest
{
StreamName = kinesisStreamName,
PartitionKey = "weather",
Data = memoryStream
};
var response = kinesisClient.PutRecordAsync(requestRecord).Result;
if ((int)response.HttpStatusCode == 200)
{
Console.WriteLine($"Successfully sent record to Kinesis. Sequence number: {response.SequenceNumber}. Next event in {timeDelay}(s)");
}
else
{
Console.Write($"Failed on: {dataAsJson}");
}
}
Thread.Sleep(timeDelay * 1000);
}
}
catch (AmazonClientException amazonEx)
{
Console.WriteLine("Opps! Looks like you don't have an Amazon cli setup or incorrect Kinesis stream name. Update appsettings.json in this build directory. Check the exception below for more details.");
Console.Write(amazonEx.GetBaseException().ToString());
Console.ReadLine();
throw;
}
catch (Exception ex)
{
Console.Write(ex.GetBaseException().ToString());
Console.ReadLine();
throw;
}
finally {
kinesisClient?.Dispose();
}
}
/// <summary>
/// Sends sensor data to AWS Kinesis Firehose.
/// </summary>
/// <param name="temp">Temperature value</param>
/// <param name="hum">Humidity value</param>
/// <param name="bat">Battery level</param>
private async void SendMessageToAWSFirehose(double temp, uint hum, uint bat)
{
// Check that the client is not null (might be if initialization failed)
if (_client != null)
{
PutRecordRequest req = new PutRecordRequest();
req.DeliveryStreamName = AWS_DELIVERY_STREAM;
// Create the message
// Message format: ISO timestamp, temperature, humidity, battery
String data = String.Format("{0},{1},{2},{3}\n", DateTime.UtcNow.ToString("u"), temp, hum, bat);
try
{
var record = new Record
{
Data = new MemoryStream(UTF8Encoding.UTF8.GetBytes(data))
};
req.Record = record;
await _client.PutRecordAsync(req);
}
catch (Exception e)
{
// Notify the user
await this.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () =>
{
statusText.Text = string.Format("Error sending data to cloud: {0}", e.ToString());
});
}
}
}
private async Task WriteToStream()
{
const string myStreamName = "myTestStream";
const int myStreamSize = 1;
try
{
var createStreamRequest = new CreateStreamRequest();
createStreamRequest.StreamName = myStreamName;
createStreamRequest.ShardCount = myStreamSize;
var createStreamReq = createStreamRequest;
var existingStreams = await kinesisClient.ListStreamsAsync();
if (!existingStreams.StreamNames.Contains(myStreamName))
{
var CreateStreamResponse = await kinesisClient.CreateStreamAsync(createStreamReq);
Console.WriteLine("Created Stream : " + myStreamName);
}
}
catch (ResourceInUseException)
{
Console.Error.WriteLine("Producer is quitting without creating stream " + myStreamName +
" to put records into as a stream of the same name already exists.");
Environment.Exit(1);
}
await WaitForStreamToBecomeAvailableAsync(myStreamName);
Console.Error.WriteLine("Putting records in stream : " + myStreamName);
// Write 10 UTF-8 encoded records to the stream.
for (int j = 0; j < 10; ++j)
{
byte[] dataAsBytes = Encoding.UTF8.GetBytes("testdata-" + j);
using (MemoryStream memoryStream = new MemoryStream(dataAsBytes))
{
try
{
PutRecordRequest requestRecord = new PutRecordRequest();
requestRecord.StreamName = myStreamName;
requestRecord.PartitionKey = "url-response-times";
requestRecord.Data = memoryStream;
PutRecordResponse responseRecord =
await kinesisClient.PutRecordAsync(requestRecord);
Console.WriteLine("Successfully sent record to Kinesis. Sequence number: {0}",
responseRecord.SequenceNumber);
}
catch (Exception ex)
{
Console.WriteLine("Failed to send record to Kinesis. Exception: {0}", ex.Message);
}
}
}
Console.ReadLine();
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecord operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecord operation on AmazonKinesisFirehoseClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndPutRecord
/// operation.</returns>
public IAsyncResult BeginPutRecord(PutRecordRequest request, AsyncCallback callback, object state)
{
var marshaller = new PutRecordRequestMarshaller();
var unmarshaller = PutRecordResponseUnmarshaller.Instance;
return(BeginInvoke <PutRecordRequest>(request, marshaller, unmarshaller,
callback, state));
}
/// <summary>
/// Writes a single data record into an Amazon Kinesis Firehose delivery stream. To write
/// multiple data records into a delivery stream, use <a>PutRecordBatch</a>. Applications
/// using these operations are referred to as producers.
///
///
/// <para>
/// By default, each delivery stream can take in up to 2,000 transactions per second,
/// 5,000 records per second, or 5 MB per second. Note that if you use <a>PutRecord</a>
/// and <a>PutRecordBatch</a>, the limits are an aggregate across these two operations
/// for each delivery stream. For more information about limits and how to request an
/// increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Firehose Limits</a>.
/// </para>
///
/// <para>
/// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>.
/// The data record consists of a data blob that can be up to 1,000 KB in size, and any
/// kind of data, for example, a segment from a log file, geographic location data, website
/// clickstream data, and so on.
/// </para>
///
/// <para>
/// Kinesis Firehose buffers records before delivering them to the destination. To disambiguate
/// the data blobs at the destination, a common solution is to use delimiters in the data,
/// such as a newline (<code>\n</code>) or some other character unique within the data.
/// This allows the consumer application to parse individual data items when reading the
/// data from the destination.
/// </para>
///
/// <para>
/// The <a>PutRecord</a> operation returns a <b>RecordId</b>, which is a unique string
/// assigned to each record. Producer applications can use this ID for purposes such as
/// auditability and investigation.
/// </para>
///
/// <para>
/// If the <a>PutRecord</a> operation throws a <b>ServiceUnavailableException</b>, back
/// off and retry. If the exception persists, it is possible that the throughput limits
/// have been exceeded for the delivery stream.
/// </para>
///
/// <para>
/// Data records sent to Kinesis Firehose are stored for 24 hours from the time they are
/// added to a delivery stream as it attempts to send the records to the destination.
/// If the destination is unreachable for more than 24 hours, the data is no longer available.
/// </para>
/// </summary>
/// <param name="deliveryStreamName">The name of the delivery stream.</param>
/// <param name="record">The record.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
///
/// <returns>The response from the PutRecord service method, as returned by KinesisFirehose.</returns>
/// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException">
/// The specified input parameter has a value that is not valid.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException">
/// The specified resource could not be found.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException">
/// The service is unavailable, back off and retry the operation. If you continue to see
/// the exception, throughput limits for the delivery stream may have been exceeded. For
/// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Firehose Limits</a>.
/// </exception>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/firehose-2015-08-04/PutRecord">REST API Reference for PutRecord Operation</seealso>
public virtual Task <PutRecordResponse> PutRecordAsync(string deliveryStreamName, Record record, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var request = new PutRecordRequest();
request.DeliveryStreamName = deliveryStreamName;
request.Record = record;
return(PutRecordAsync(request, cancellationToken));
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecord operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecord operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/firehose-2015-08-04/PutRecord">REST API Reference for PutRecord Operation</seealso>
public virtual Task <PutRecordResponse> PutRecordAsync(PutRecordRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new PutRecordRequestMarshaller();
var unmarshaller = PutRecordResponseUnmarshaller.Instance;
return(InvokeAsync <PutRecordRequest, PutRecordResponse>(request, marshaller,
unmarshaller, cancellationToken));
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecord operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecord operation on AmazonSageMakerFeatureStoreRuntimeClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndPutRecord
/// operation.</returns>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/sagemaker-featurestore-runtime-2020-07-01/PutRecord">REST API Reference for PutRecord Operation</seealso>
public virtual IAsyncResult BeginPutRecord(PutRecordRequest request, AsyncCallback callback, object state)
{
var options = new InvokeOptions();
options.RequestMarshaller = PutRecordRequestMarshaller.Instance;
options.ResponseUnmarshaller = PutRecordResponseUnmarshaller.Instance;
return BeginInvoke(request, options, callback, state);
}
/// <summary>
/// Used for data ingestion into the <code>FeatureStore</code>. The <code>PutRecord</code>
/// API writes to both the <code>OnlineStore</code> and <code>OfflineStore</code>. If
/// the record is the latest record for the <code>recordIdentifier</code>, the record
/// is written to both the <code>OnlineStore</code> and <code>OfflineStore</code>. If
/// the record is a historic record, it is written only to the <code>OfflineStore</code>.
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the PutRecord service method.</param>
///
/// <returns>The response from the PutRecord service method, as returned by SageMakerFeatureStoreRuntime.</returns>
/// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.AccessForbiddenException">
/// You do not have permission to perform an action.
/// </exception>
/// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.InternalFailureException">
/// An internal failure occurred. Try your request again. If the problem persists, contact
/// AWS customer support.
/// </exception>
/// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.ServiceUnavailableException">
/// The service is currently unavailable.
/// </exception>
/// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.ValidationErrorException">
/// There was an error validating your request.
/// </exception>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/sagemaker-featurestore-runtime-2020-07-01/PutRecord">REST API Reference for PutRecord Operation</seealso>
public virtual PutRecordResponse PutRecord(PutRecordRequest request)
{
var options = new InvokeOptions();
options.RequestMarshaller = PutRecordRequestMarshaller.Instance;
options.ResponseUnmarshaller = PutRecordResponseUnmarshaller.Instance;
return Invoke<PutRecordResponse>(request, options);
}
/// <summary>
/// Writes a single data record into an Amazon Kinesis Data Firehose delivery stream.
/// To write multiple data records into a delivery stream, use <a>PutRecordBatch</a>.
/// Applications using these operations are referred to as producers.
///
///
/// <para>
/// By default, each delivery stream can take in up to 2,000 transactions per second,
/// 5,000 records per second, or 5 MB per second. If you use <a>PutRecord</a> and <a>PutRecordBatch</a>,
/// the limits are an aggregate across these two operations for each delivery stream.
/// For more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Data Firehose Limits</a>.
/// </para>
///
/// <para>
/// You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>.
/// The data record consists of a data blob that can be up to 1,000 KB in size, and any
/// kind of data. For example, it can be a segment from a log file, geographic location
/// data, website clickstream data, and so on.
/// </para>
///
/// <para>
/// Kinesis Data Firehose buffers records before delivering them to the destination. To
/// disambiguate the data blobs at the destination, a common solution is to use delimiters
/// in the data, such as a newline (<code>\n</code>) or some other character unique within
/// the data. This allows the consumer application to parse individual data items when
/// reading the data from the destination.
/// </para>
///
/// <para>
/// The <code>PutRecord</code> operation returns a <code>RecordId</code>, which is a unique
/// string assigned to each record. Producer applications can use this ID for purposes
/// such as auditability and investigation.
/// </para>
///
/// <para>
/// If the <code>PutRecord</code> operation throws a <code>ServiceUnavailableException</code>,
/// back off and retry. If the exception persists, it is possible that the throughput
/// limits have been exceeded for the delivery stream.
/// </para>
///
/// <para>
/// Data records sent to Kinesis Data Firehose are stored for 24 hours from the time they
/// are added to a delivery stream as it tries to send the records to the destination.
/// If the destination is unreachable for more than 24 hours, the data is no longer available.
/// </para>
/// <important>
/// <para>
/// Don't concatenate two or more base64 strings to form the data fields of your records.
/// Instead, concatenate the raw data, then perform base64 encoding.
/// </para>
/// </important>
/// </summary>
/// <param name="deliveryStreamName">The name of the delivery stream.</param>
/// <param name="record">The record.</param>
/// <param name="callback">An Action delegate that is invoked when the operation completes.</param>
/// <param name="options">
/// A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.
/// </param>
///
/// <returns>The response from the PutRecord service method, as returned by KinesisFirehose.</returns>
/// <exception cref="Amazon.KinesisFirehose.Model.InvalidArgumentException">
/// The specified input parameter has a value that is not valid.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ResourceNotFoundException">
/// The specified resource could not be found.
/// </exception>
/// <exception cref="Amazon.KinesisFirehose.Model.ServiceUnavailableException">
/// The service is unavailable. Back off and retry the operation. If you continue to see
/// the exception, throughput limits for the delivery stream may have been exceeded. For
/// more information about limits and how to request an increase, see <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
/// Kinesis Data Firehose Limits</a>.
/// </exception>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/firehose-2015-08-04/PutRecord">REST API Reference for PutRecord Operation</seealso>
public virtual void PutRecordAsync(string deliveryStreamName, Record record, AmazonServiceCallback <PutRecordRequest, PutRecordResponse> callback, AsyncOptions options = null)
{
var request = new PutRecordRequest();
request.DeliveryStreamName = deliveryStreamName;
request.Record = record;
PutRecordAsync(request, callback, options);
}
/// <summary>
/// This method verifies your credentials, creates a Kinesis stream, waits for the stream
/// to become active, then puts 10 records in it, and (optionally) deletes the stream.
/// </summary>
public async static Task Main(string[] args)
{
const string myStreamName = "myTestStream";
const int myStreamSize = 1;
try
{
var createStreamRequest = new CreateStreamRequest();
createStreamRequest.StreamName = myStreamName;
createStreamRequest.ShardCount = myStreamSize;
var createStreamReq = createStreamRequest;
await kinesisClient.CreateStreamAsync(createStreamReq);
Console.Error.WriteLine("Created Stream : " + myStreamName);
}
catch (ResourceInUseException)
{
Console.Error.WriteLine("Producer is quitting without creating stream " + myStreamName +
" to put records into as a stream of the same name already exists.");
Environment.Exit(1);
}
await WaitForStreamToBecomeAvailable(myStreamName);
Console.Error.WriteLine("Putting records in stream : " + myStreamName);
// Write 10 UTF-8 encoded records to the stream.
for (int j = 0; j < 10; ++j)
{
PutRecordRequest requestRecord = new PutRecordRequest();
requestRecord.StreamName = myStreamName;
requestRecord.Data = new MemoryStream(Encoding.UTF8.GetBytes("testData-" + j));
requestRecord.PartitionKey = "partitionKey-" + j;
var putResult = await kinesisClient.PutRecordAsync(requestRecord);
Console.Error.WriteLine(
String.Format("Successfully putrecord {0}:\n\t partition key = {1,15}, shard ID = {2}",
j, requestRecord.PartitionKey, putResult.ShardId));
}
// Uncomment the following if you wish to delete the stream here.
//Console.Error.WriteLine("Deleting stream : " + myStreamName);
//DeleteStreamRequest deleteStreamReq = new DeleteStreamRequest();
//deleteStreamReq.StreamName = myStreamName;
//try
//{
// kinesisClient.DeleteStream(deleteStreamReq);
// Console.Error.WriteLine("Stream is now being deleted : " + myStreamName);
//}
//catch (ResourceNotFoundException ex)
//
// Console.Error.WriteLine("Stream could not be found; " + ex);
//}
//catch (AmazonClientException ex)
//{
// Console.Error.WriteLine("Error deleting stream; " + ex);
//}
}
/// <summary>
/// Used for data ingestion into the <code>FeatureStore</code>. The <code>PutRecord</code>
/// API writes to both the <code>OnlineStore</code> and <code>OfflineStore</code>. If
/// the record is the latest record for the <code>recordIdentifier</code>, the record
/// is written to both the <code>OnlineStore</code> and <code>OfflineStore</code>. If
/// the record is a historic record, it is written only to the <code>OfflineStore</code>.
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the PutRecord service method.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
///
/// <returns>The response from the PutRecord service method, as returned by SageMakerFeatureStoreRuntime.</returns>
/// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.AccessForbiddenException">
/// You do not have permission to perform an action.
/// </exception>
/// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.InternalFailureException">
/// An internal failure occurred. Try your request again. If the problem persists, contact
/// AWS customer support.
/// </exception>
/// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.ServiceUnavailableException">
/// The service is currently unavailable.
/// </exception>
/// <exception cref="Amazon.SageMakerFeatureStoreRuntime.Model.ValidationErrorException">
/// There was an error validating your request.
/// </exception>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/sagemaker-featurestore-runtime-2020-07-01/PutRecord">REST API Reference for PutRecord Operation</seealso>
public virtual Task <PutRecordResponse> PutRecordAsync(PutRecordRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var options = new InvokeOptions();
options.RequestMarshaller = PutRecordRequestMarshaller.Instance;
options.ResponseUnmarshaller = PutRecordResponseUnmarshaller.Instance;
return(InvokeAsync <PutRecordResponse>(request, options, cancellationToken));
}
public void Send(Dictionary <string, object> logEvent)
{
var requestRecord = new PutRecordRequest
{
StreamName = StreamName,
Data = new MemoryStream(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(logEvent))),
PartitionKey = AppName
};
Task.Run(() => _client.PutRecordAsync(requestRecord));
}
public async Task <PutRecordResponse> PutKinesisRecord(string data, string partitionKey)
{
var putRecordRequest = new PutRecordRequest
{
StreamName = _StreamName,
PartitionKey = partitionKey,
Data = new MemoryStream(Encoding.UTF8.GetBytes(data))
};
return(await _AmazonKinesisClient.PutRecordAsync(putRecordRequest));
}
public async Task SendMessageAsync(TMessage message)
{
PutRecordRequest requestRecord = new PutRecordRequest();
requestRecord.StreamName = streamName;
requestRecord.Data = new MemoryStream(serializer.SerializeToArray(message));
requestRecord.PartitionKey = partitionKeyFunc(message);
await kinesisClient.PutRecordAsync(requestRecord);
}
public void LogRecord(Record record)
{
byte[] dataAsBytes = Encoding.UTF8.GetBytes(record.ToJson());
using (MemoryStream memoryStream = new MemoryStream(dataAsBytes))
{
PutRecordRequest requestRecord = new PutRecordRequest();
requestRecord.StreamName = StreamName;
requestRecord.PartitionKey = PartitionKey;
requestRecord.Data = memoryStream;
PutRecordResponse responseRecord = KinesisClient.PutRecord(requestRecord);
}
}
public async void Accept(CommandChunk chunk)
{
var bytes = _serializer.Serialize(chunk);
var request = new PutRecordRequest
{
SequenceNumberForOrdering = chunk.EngineSequenceNumber.ToString(),
StreamName = _streamName,
Data = new MemoryStream(bytes),
PartitionKey = chunk.Engine.ToString() //todo: ordering guarantees?
};
var response = await _client.PutRecordAsync(request);
var number = response.SequenceNumber;
}
/// <summary>
/// <para>This operation puts a data record into an Amazon Kinesis stream from a producer. This operation must be called to send data from the
/// producer into the Amazon Kinesis stream for real-time ingestion and subsequent processing. The <c>PutRecord</c> operation requires the name
/// of the stream that captures, stores, and transports the data; a partition key; and the data blob itself. The data blob could be a segment
/// from a log file, geographic/location data, website clickstream data, or any other data type.</para> <para>The partition key is used to
/// distribute data across shards. Amazon Kinesis segregates the data records that belong to a data stream into multiple shards, using the
/// partition key associated with each data record to determine which shard a given data record belongs to. </para> <para>Partition keys are
/// Unicode strings, with a maximum length limit of 256 bytes. An MD5 hash function is used to map partition keys to 128-bit integer values and
/// to map associated data records to shards using the hash key ranges of the shards. You can override hashing the partition key to determine
/// the shard by explicitly specifying a hash value using the <c>ExplicitHashKey</c> parameter. For more information, see the <a href="http://docs.aws.amazon.com/kinesis/latest/dev/">Amazon Kinesis Developer Guide</a> .</para> <para> <c>PutRecord</c> returns the shard
/// ID of where the data record was placed and the sequence number that was assigned to the data record.</para> <para>Sequence numbers generally
/// increase over time. To guarantee strictly increasing ordering, use the <c>SequenceNumberForOrdering</c> parameter. For more information, see
/// the <a href="http://docs.aws.amazon.com/kinesis/latest/dev/">Amazon Kinesis Developer Guide</a> .</para> <para>If a <c>PutRecord</c>
/// request cannot be processed because of insufficient provisioned throughput on the shard involved in the request, <c>PutRecord</c> throws
/// <c>ProvisionedThroughputExceededException</c> . </para> <para>Data records are accessible for only 24 hours from the time that they are
/// added to an Amazon Kinesis stream.</para>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecord service method on AmazonKinesis.</param>
///
/// <returns>The response from the PutRecord service method, as returned by AmazonKinesis.</returns>
///
/// <exception cref="T:Amazon.Kinesis.Model.ProvisionedThroughputExceededException" />
/// <exception cref="T:Amazon.Kinesis.Model.InvalidArgumentException" />
/// <exception cref="T:Amazon.Kinesis.Model.ResourceNotFoundException" />
public PutRecordResponse PutRecord(PutRecordRequest request)
{
var task = PutRecordAsync(request);
try
{
return(task.Result);
}
catch (AggregateException e)
{
ExceptionDispatchInfo.Capture(e.InnerException).Throw();
return(null);
}
}
public async Task RecordAsync(Activity record, string partitionKey)
{
var bytes = Serialize(record);
using (var ms = new MemoryStream(bytes))
{
var putRecordRequest = new PutRecordRequest
{
StreamName = _streamName,
Data = ms,
PartitionKey = partitionKey
};
await _kinesis.PutRecordAsync(putRecordRequest);
}
}
public async void Handle(CommandChunk chunk)
{
var bytes = _serializer.Serialize(chunk);
var request = new PutRecordRequest
{
SequenceNumberForOrdering = chunk.LocalSequenceNumber.ToString(),
StreamName = _streamName,
Data = new MemoryStream(bytes),
PartitionKey = chunk.PartitionKey
};
var response = await _client.PutRecordAsync(request);
chunk.GlobalSequenceNumber = response.SequenceNumber;
}
public async Task <string> PutAsync(Message message, string streamName)
{
var json = JsonConvert.SerializeObject(message);
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)))
{
var putRecordRequest = new PutRecordRequest {
DeliveryStreamName = streamName, Record = new Record {
Data = stream
}
};
var response = await _client.PutRecordAsync(putRecordRequest);
return(response.RecordId);
}
}