/// <summary>
/// This method verifies your credentials, creates a Kinesis stream, waits for the stream
/// to become active, then puts 10 records in it, and (optionally) deletes the stream.
/// </summary>
public static void Main(string[] args)
{
const string myStreamName = "myTestStream";
const int myStreamSize = 1;
try
{
var createStreamRequest = new CreateStreamRequest();
createStreamRequest.StreamName = myStreamName;
createStreamRequest.ShardCount = myStreamSize;
var createStreamReq = createStreamRequest;
kinesisClient.CreateStream(createStreamReq);
Console.Error.WriteLine("Created Stream : " + myStreamName);
}
catch (ResourceInUseException)
{
Console.Error.WriteLine("Producer is quitting without creating stream " + myStreamName +
" to put records into as a stream of the same name already exists.");
Environment.Exit(1);
}
WaitForStreamToBecomeAvailable(myStreamName);
Console.Error.WriteLine("Putting records in stream : " + myStreamName);
// Write 10 UTF-8 encoded records to the stream.
for (int j = 0; j < 10; ++j)
{
PutRecordRequest requestRecord = new PutRecordRequest();
requestRecord.StreamName = myStreamName;
requestRecord.Data = new MemoryStream(Encoding.UTF8.GetBytes("testData-" + j));
requestRecord.PartitionKey = "partitionKey-" + j;
PutRecordResult putResult = kinesisClient.PutRecord(requestRecord);
Console.Error.WriteLine(
String.Format("Successfully putrecord {0}:\n\t partition key = {1,15}, shard ID = {2}",
j, requestRecord.PartitionKey, putResult.ShardId));
}
// Uncomment the following if you wish to delete the stream here.
//Console.Error.WriteLine("Deleting stream : " + myStreamName);
//DeleteStreamRequest deleteStreamReq = new DeleteStreamRequest();
//deleteStreamReq.StreamName = myStreamName;
//try
//{
// kinesisClient.DeleteStream(deleteStreamReq);
// Console.Error.WriteLine("Stream is now being deleted : " + myStreamName);
//}
//catch (ResourceNotFoundException ex)
//
// Console.Error.WriteLine("Stream could not be found; " + ex);
//}
//catch (AmazonClientException ex)
//{
// Console.Error.WriteLine("Error deleting stream; " + ex);
//}
}
public async Task<IHttpActionResult> Post()
{
var o = new
{
Message = "Hello World",
Author = "David Judd"
};
//convert to byte array in prep for adding to stream
byte[] oByte = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(o));
//create client that pulls creds from web.config and takes in Kinesis config
var client = new AmazonKinesisClient(Config);
using (MemoryStream ms = new MemoryStream(oByte))
{
//create put request
PutRecordRequest requestRecord = new PutRecordRequest();
//list name of Kinesis stream
requestRecord.StreamName = "shomi_dev";
//give partition key that is used to place record in particular shard
requestRecord.PartitionKey = DateTime.Now.Ticks.ToString();
//add record as memorystream
requestRecord.Data = ms;
//PUT the record to Kinesis
var response = await client.PutRecordAsync(requestRecord);
return Ok(new
{
seq = response.SequenceNumber
});
}
}
private static void SendWebTransactionsToQueue(List <WebTransaction> transactions)
{
Amazon.Kinesis.AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1;
AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
String kinesisStreamName = ConfigurationManager.AppSettings["KinesisStreamName"];
foreach (WebTransaction wt in transactions)
{
string dataAsJson = JsonConvert.SerializeObject(wt);
byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson);
using (MemoryStream memoryStream = new MemoryStream(dataAsBytes))
{
try
{
//PutRecordsRequestEntry requestRecord = new PutRecordRequest();
Amazon.Kinesis.Model.PutRecordRequest requestRecord = new Amazon.Kinesis.Model.PutRecordRequest();
requestRecord.StreamName = kinesisStreamName;
requestRecord.PartitionKey = "url-response-times";
requestRecord.Data = memoryStream;
Amazon.Kinesis.Model.PutRecordResponse responseRecord = kinesisClient.PutRecord(requestRecord);
Console.WriteLine("Successfully sent record {0} to Kinesis. Sequence number: {1}", wt.Url, responseRecord.SequenceNumber);
}
catch (Exception ex)
{
Console.WriteLine("Failed to send record {0} to Kinesis. Exception: {1}", wt.Url, ex.Message);
}
}
}
}
PutRecordResponse PutRecord (AmazonKinesisClient client)
{
var request = new PutRecordRequest ();
request.StreamName = "Test1";
request.Data = new System.IO.MemoryStream (new byte[] { 123 });
request.PartitionKey = "p1337";
return client.PutRecord (request);
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecord operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecord operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task<PutRecordResponse> PutRecordAsync(PutRecordRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new PutRecordRequestMarshaller();
var unmarshaller = PutRecordResponseUnmarshaller.Instance;
return InvokeAsync<PutRecordRequest,PutRecordResponse>(request, marshaller,
unmarshaller, cancellationToken);
}
internal PutRecordResponse PutRecord(PutRecordRequest request)
{
var marshaller = new PutRecordRequestMarshaller();
var unmarshaller = PutRecordResponseUnmarshaller.Instance;
return Invoke<PutRecordRequest,PutRecordResponse>(request, marshaller, unmarshaller);
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecord operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecord operation on AmazonKinesisClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndPutRecord
/// operation.</returns>
public IAsyncResult BeginPutRecord(PutRecordRequest request, AsyncCallback callback, object state)
{
var marshaller = new PutRecordRequestMarshaller();
var unmarshaller = PutRecordResponseUnmarshaller.Instance;
return BeginInvoke<PutRecordRequest>(request, marshaller, unmarshaller,
callback, state);
}
/// <summary>
/// <para>This operation puts a data record into an Amazon Kinesis stream from a producer. This operation must be called to send data from the
/// producer into the Amazon Kinesis stream for real-time ingestion and subsequent processing. The <c>PutRecord</c> operation requires the name
/// of the stream that captures, stores, and transports the data; a partition key; and the data blob itself. The data blob could be a segment
/// from a log file, geographic/location data, website clickstream data, or any other data type.</para> <para>The partition key is used to
/// distribute data across shards. Amazon Kinesis segregates the data records that belong to a data stream into multiple shards, using the
/// partition key associated with each data record to determine which shard a given data record belongs to. </para> <para>Partition keys are
/// Unicode strings, with a maximum length limit of 256 bytes. An MD5 hash function is used to map partition keys to 128-bit integer values and
/// to map associated data records to shards using the hash key ranges of the shards. You can override hashing the partition key to determine
/// the shard by explicitly specifying a hash value using the <c>ExplicitHashKey</c> parameter. For more information, see the <a href="http://docs.aws.amazon.com/kinesis/latest/dev/">Amazon Kinesis Developer Guide</a> .</para> <para> <c>PutRecord</c> returns the shard
/// ID of where the data record was placed and the sequence number that was assigned to the data record.</para> <para>Sequence numbers generally
/// increase over time. To guarantee strictly increasing ordering, use the <c>SequenceNumberForOrdering</c> parameter. For more information, see
/// the <a href="http://docs.aws.amazon.com/kinesis/latest/dev/">Amazon Kinesis Developer Guide</a> .</para> <para>If a <c>PutRecord</c>
/// request cannot be processed because of insufficient provisioned throughput on the shard involved in the request, <c>PutRecord</c> throws
/// <c>ProvisionedThroughputExceededException</c> . </para> <para>Data records are accessible for only 24 hours from the time that they are
/// added to an Amazon Kinesis stream.</para>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecord service method on AmazonKinesis.</param>
///
/// <returns>The response from the PutRecord service method, as returned by AmazonKinesis.</returns>
///
/// <exception cref="T:Amazon.Kinesis.Model.ProvisionedThroughputExceededException" />
/// <exception cref="T:Amazon.Kinesis.Model.InvalidArgumentException" />
/// <exception cref="T:Amazon.Kinesis.Model.ResourceNotFoundException" />
public PutRecordResponse PutRecord(PutRecordRequest request)
{
var task = PutRecordAsync(request);
try
{
return task.Result;
}
catch(AggregateException e)
{
ExceptionDispatchInfo.Capture(e.InnerException).Throw();
return null;
}
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecord operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecord operation on AmazonKinesisClient.</param>
/// <param name="callback">An Action delegate that is invoked when the operation completes.</param>
/// <param name="options">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
public void PutRecordAsync(PutRecordRequest request, AmazonServiceCallback<PutRecordRequest, PutRecordResponse> callback, AsyncOptions options = null)
{
options = options == null?new AsyncOptions():options;
var marshaller = new PutRecordRequestMarshaller();
var unmarshaller = PutRecordResponseUnmarshaller.Instance;
Action<AmazonWebServiceRequest, AmazonWebServiceResponse, Exception, AsyncOptions> callbackHelper = null;
if(callback !=null )
callbackHelper = (AmazonWebServiceRequest req, AmazonWebServiceResponse res, Exception ex, AsyncOptions ao) => {
AmazonServiceResult<PutRecordRequest,PutRecordResponse> responseObject
= new AmazonServiceResult<PutRecordRequest,PutRecordResponse>((PutRecordRequest)req, (PutRecordResponse)res, ex , ao.State);
callback(responseObject);
};
BeginInvoke<PutRecordRequest>(request, marshaller, unmarshaller, options, callbackHelper);
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecord operation.
/// <seealso cref="Amazon.Kinesis.IAmazonKinesis.PutRecord"/>
/// </summary>
///
/// <param name="putRecordRequest">Container for the necessary parameters to execute the PutRecord operation on AmazonKinesis.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndPutRecord
/// operation.</returns>
public IAsyncResult BeginPutRecord(PutRecordRequest putRecordRequest, AsyncCallback callback, object state)
{
return invokePutRecord(putRecordRequest, callback, state, false);
}
/// <summary>
/// <para>This operation puts a data record into an Amazon Kinesis stream from a producer. This operation must be called to send data from the
/// producer into the Amazon Kinesis stream for real-time ingestion and subsequent processing. The <c>PutRecord</c> operation requires the name
/// of the stream that captures, stores, and transports the data; a partition key; and the data blob itself. The data blob could be a segment
/// from a log file, geographic/location data, website clickstream data, or any other data type.</para> <para>The partition key is used to
/// distribute data across shards. Amazon Kinesis segregates the data records that belong to a data stream into multiple shards, using the
/// partition key associated with each data record to determine which shard a given data record belongs to. </para> <para>Partition keys are
/// Unicode strings, with a maximum length limit of 256 bytes. An MD5 hash function is used to map partition keys to 128-bit integer values and
/// to map associated data records to shards using the hash key ranges of the shards. You can override hashing the partition key to determine
/// the shard by explicitly specifying a hash value using the <c>ExplicitHashKey</c> parameter. For more information, see the <a
/// href="http://docs.aws.amazon.com/kinesis/latest/dev/" >Amazon Kinesis Developer Guide</a> .</para> <para> <c>PutRecord</c> returns the shard
/// ID of where the data record was placed and the sequence number that was assigned to the data record.</para> <para>Sequence numbers generally
/// increase over time. To guarantee strictly increasing ordering, use the <c>SequenceNumberForOrdering</c> parameter. For more information, see
/// the <a href="http://docs.aws.amazon.com/kinesis/latest/dev/" >Amazon Kinesis Developer Guide</a> .</para> <para>If a <c>PutRecord</c>
/// request cannot be processed because of insufficient provisioned throughput on the shard involved in the request, <c>PutRecord</c> throws
/// <c>ProvisionedThroughputExceededException</c> . </para> <para>Data records are accessible for only 24 hours from the time that they are
/// added to an Amazon Kinesis stream.</para>
/// </summary>
///
/// <param name="putRecordRequest">Container for the necessary parameters to execute the PutRecord service method on AmazonKinesis.</param>
///
/// <returns>The response from the PutRecord service method, as returned by AmazonKinesis.</returns>
///
/// <exception cref="ProvisionedThroughputExceededException"/>
/// <exception cref="InvalidArgumentException"/>
/// <exception cref="ResourceNotFoundException"/>
public PutRecordResponse PutRecord(PutRecordRequest putRecordRequest)
{
IAsyncResult asyncResult = invokePutRecord(putRecordRequest, null, null, true);
return EndPutRecord(asyncResult);
}
private Amazon.Kinesis.Model.PutRecordResponse CallAWSServiceOperation(IAmazonKinesis client, Amazon.Kinesis.Model.PutRecordRequest request)
{
Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Kinesis", "PutRecord");
try
{
#if DESKTOP
return(client.PutRecord(request));
#elif CORECLR
return(client.PutRecordAsync(request).GetAwaiter().GetResult());
#else
#error "Unknown build edition"
#endif
}
catch (AmazonServiceException exc)
{
var webException = exc.InnerException as System.Net.WebException;
if (webException != null)
{
throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException);
}
throw;
}
}
public object Execute(ExecutorContext context)
{
System.IO.MemoryStream _DataStream = null;
try
{
var cmdletContext = context as CmdletContext;
// create request
var request = new Amazon.Kinesis.Model.PutRecordRequest();
if (cmdletContext.Data != null)
{
_DataStream = new System.IO.MemoryStream(cmdletContext.Data);
request.Data = _DataStream;
}
if (cmdletContext.ExplicitHashKey != null)
{
request.ExplicitHashKey = cmdletContext.ExplicitHashKey;
}
if (cmdletContext.PartitionKey != null)
{
request.PartitionKey = cmdletContext.PartitionKey;
}
if (cmdletContext.SequenceNumberForOrdering != null)
{
request.SequenceNumberForOrdering = cmdletContext.SequenceNumberForOrdering;
}
if (cmdletContext.StreamName != null)
{
request.StreamName = cmdletContext.StreamName;
}
CmdletOutput output;
// issue call
var client = Client ?? CreateClient(_CurrentCredentials, _RegionEndpoint);
try
{
var response = CallAWSServiceOperation(client, request);
object pipelineOutput = null;
pipelineOutput = cmdletContext.Select(response, this);
output = new CmdletOutput
{
PipelineOutput = pipelineOutput,
ServiceResponse = response
};
}
catch (Exception e)
{
output = new CmdletOutput {
ErrorResponse = e
};
}
return(output);
}
finally
{
if (_DataStream != null)
{
_DataStream.Dispose();
}
}
}
internal PutRecordResponse PutRecord(PutRecordRequest request)
{
var task = PutRecordAsync(request);
try
{
return task.Result;
}
catch(AggregateException e)
{
throw e.InnerException;
}
}
IAsyncResult invokePutRecord(PutRecordRequest putRecordRequest, AsyncCallback callback, object state, bool synchronized)
{
IRequest irequest = new PutRecordRequestMarshaller().Marshall(putRecordRequest);
var unmarshaller = PutRecordResponseUnmarshaller.GetInstance();
AsyncResult result = new AsyncResult(irequest, callback, state, synchronized, signer, unmarshaller);
Invoke(result);
return result;
}
/// <summary>
/// <para>This operation puts a data record into an Amazon Kinesis stream from a producer. This operation must be called to send data from the
/// producer into the Amazon Kinesis stream for real-time ingestion and subsequent processing. The <c>PutRecord</c> operation requires the name
/// of the stream that captures, stores, and transports the data; a partition key; and the data blob itself. The data blob could be a segment
/// from a log file, geographic/location data, website clickstream data, or any other data type.</para> <para>The partition key is used to
/// distribute data across shards. Amazon Kinesis segregates the data records that belong to a data stream into multiple shards, using the
/// partition key associated with each data record to determine which shard a given data record belongs to. </para> <para>Partition keys are
/// Unicode strings, with a maximum length limit of 256 bytes. An MD5 hash function is used to map partition keys to 128-bit integer values and
/// to map associated data records to shards using the hash key ranges of the shards. You can override hashing the partition key to determine
/// the shard by explicitly specifying a hash value using the <c>ExplicitHashKey</c> parameter. For more information, see the <a href="http://docs.aws.amazon.com/kinesis/latest/dev/">Amazon Kinesis Developer Guide</a> .</para> <para> <c>PutRecord</c> returns the shard
/// ID of where the data record was placed and the sequence number that was assigned to the data record.</para> <para>The
/// <c>SequenceNumberForOrdering</c> sets the initial sequence number for the partition key. Later <c>PutRecord</c> requests to the same
/// partition key (from the same client) will automatically increase from <c>SequenceNumberForOrdering</c> , ensuring strict sequential
/// ordering.</para> <para>If a <c>PutRecord</c> request cannot be processed because of insufficient provisioned throughput on the shard
/// involved in the request, <c>PutRecord</c> throws <c>ProvisionedThroughputExceededException</c> . </para> <para>Data records are accessible
/// for only 24 hours from the time that they are added to an Amazon Kinesis stream.</para>
/// </summary>
///
/// <param name="putRecordRequest">Container for the necessary parameters to execute the PutRecord service method on AmazonKinesis.</param>
///
/// <returns>The response from the PutRecord service method, as returned by AmazonKinesis.</returns>
///
/// <exception cref="T:Amazon.Kinesis.Model.ProvisionedThroughputExceededException" />
/// <exception cref="T:Amazon.Kinesis.Model.InvalidArgumentException" />
/// <exception cref="T:Amazon.Kinesis.Model.ResourceNotFoundException" />
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
public Task<PutRecordResponse> PutRecordAsync(PutRecordRequest putRecordRequest, CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new PutRecordRequestMarshaller();
var unmarshaller = PutRecordResponseUnmarshaller.GetInstance();
return Invoke<IRequest, PutRecordRequest, PutRecordResponse>(putRecordRequest, marshaller, unmarshaller, signer, cancellationToken);
}
static void Main(string[] args)
{
var o = new
{
Message = "Hello World",
Author = "David Judd"
};
//convert to byte array in prep for adding to stream
byte[] oByte = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(o));
//create config that points to AWS region
var config = new AmazonKinesisConfig();
config.RegionEndpoint = Amazon.RegionEndpoint.USEast1;
//create client that pulls creds from web.config and takes in Kinesis config
var client = new AmazonKinesisClient(config);
var sw = Stopwatch.StartNew();
Stopwatch sw2 = null;
var tasks = new List<Task<PutRecordResponse>>();
int count = int.Parse(ConfigurationManager.AppSettings["Count"]);
Console.WriteLine("Sending {0} records... One at a time...", count);
sw.Restart();
for (int i = 0; i < count; i++)
{
//System.Threading.Thread.Sleep(10);
//sw2 = Stopwatch.StartNew();
//create stream object to add to Kinesis request
using (MemoryStream ms = new MemoryStream(oByte))
{
//create put request
PutRecordRequest requestRecord = new PutRecordRequest();
//list name of Kinesis stream
requestRecord.StreamName = "shomi_dev";
//give partition key that is used to place record in particular shard
requestRecord.PartitionKey = i.ToString();
//add record as memorystream
requestRecord.Data = ms;
//PUT the record to Kinesis
var task = client.PutRecordAsync(requestRecord);
tasks.Add(task);
}
//sw2.Stop();
///Console.WriteLine("Async latency is {0}", sw2.ElapsedMilliseconds);
}
Console.WriteLine("{0} records sent... Waiting for tasks to complete...", count);
Task.WaitAll(tasks.ToArray(), -1);
sw.Stop();
foreach (var t in tasks)
{
if (t.Result.HttpStatusCode != System.Net.HttpStatusCode.OK)
{
Console.WriteLine(t.Result.HttpStatusCode);
}
}
double actionsPerSec = (double)count * 1000 / (double)sw.ElapsedMilliseconds;
Console.WriteLine("{0} requests in {1} ms. {2:0.00} requests/sec.", count, sw.ElapsedMilliseconds, actionsPerSec);
}
