/// <summary> /// This method verifies your credentials, creates a Kinesis stream, waits for the stream /// to become active, then puts 10 records in it, and (optionally) deletes the stream. /// </summary> public static void Main(string[] args) { const string myStreamName = "myTestStream"; const int myStreamSize = 1; try { var createStreamRequest = new CreateStreamRequest(); createStreamRequest.StreamName = myStreamName; createStreamRequest.ShardCount = myStreamSize; var createStreamReq = createStreamRequest; kinesisClient.CreateStream(createStreamReq); Console.Error.WriteLine("Created Stream : " + myStreamName); } catch (ResourceInUseException) { Console.Error.WriteLine("Producer is quitting without creating stream " + myStreamName + " to put records into as a stream of the same name already exists."); Environment.Exit(1); } WaitForStreamToBecomeAvailable(myStreamName); Console.Error.WriteLine("Putting records in stream : " + myStreamName); // Write 10 UTF-8 encoded records to the stream. for (int j = 0; j < 10; ++j) { PutRecordRequest requestRecord = new PutRecordRequest(); requestRecord.StreamName = myStreamName; requestRecord.Data = new MemoryStream(Encoding.UTF8.GetBytes("testData-" + j)); requestRecord.PartitionKey = "partitionKey-" + j; PutRecordResult putResult = kinesisClient.PutRecord(requestRecord); Console.Error.WriteLine( String.Format("Successfully putrecord {0}:\n\t partition key = {1,15}, shard ID = {2}", j, requestRecord.PartitionKey, putResult.ShardId)); } // Uncomment the following if you wish to delete the stream here. //Console.Error.WriteLine("Deleting stream : " + myStreamName); //DeleteStreamRequest deleteStreamReq = new DeleteStreamRequest(); //deleteStreamReq.StreamName = myStreamName; //try //{ // kinesisClient.DeleteStream(deleteStreamReq); // Console.Error.WriteLine("Stream is now being deleted : " + myStreamName); //} //catch (ResourceNotFoundException ex) // // Console.Error.WriteLine("Stream could not be found; " + ex); //} //catch (AmazonClientException ex) //{ // Console.Error.WriteLine("Error deleting stream; " + ex); //} }
public async Task<IHttpActionResult> Post() { var o = new { Message = "Hello World", Author = "David Judd" }; //convert to byte array in prep for adding to stream byte[] oByte = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(o)); //create client that pulls creds from web.config and takes in Kinesis config var client = new AmazonKinesisClient(Config); using (MemoryStream ms = new MemoryStream(oByte)) { //create put request PutRecordRequest requestRecord = new PutRecordRequest(); //list name of Kinesis stream requestRecord.StreamName = "shomi_dev"; //give partition key that is used to place record in particular shard requestRecord.PartitionKey = DateTime.Now.Ticks.ToString(); //add record as memorystream requestRecord.Data = ms; //PUT the record to Kinesis var response = await client.PutRecordAsync(requestRecord); return Ok(new { seq = response.SequenceNumber }); } }
private static void SendWebTransactionsToQueue(List <WebTransaction> transactions) { Amazon.Kinesis.AmazonKinesisConfig config = new AmazonKinesisConfig(); config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1; AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config); String kinesisStreamName = ConfigurationManager.AppSettings["KinesisStreamName"]; foreach (WebTransaction wt in transactions) { string dataAsJson = JsonConvert.SerializeObject(wt); byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson); using (MemoryStream memoryStream = new MemoryStream(dataAsBytes)) { try { //PutRecordsRequestEntry requestRecord = new PutRecordRequest(); Amazon.Kinesis.Model.PutRecordRequest requestRecord = new Amazon.Kinesis.Model.PutRecordRequest(); requestRecord.StreamName = kinesisStreamName; requestRecord.PartitionKey = "url-response-times"; requestRecord.Data = memoryStream; Amazon.Kinesis.Model.PutRecordResponse responseRecord = kinesisClient.PutRecord(requestRecord); Console.WriteLine("Successfully sent record {0} to Kinesis. Sequence number: {1}", wt.Url, responseRecord.SequenceNumber); } catch (Exception ex) { Console.WriteLine("Failed to send record {0} to Kinesis. Exception: {1}", wt.Url, ex.Message); } } } }
PutRecordResponse PutRecord (AmazonKinesisClient client) { var request = new PutRecordRequest (); request.StreamName = "Test1"; request.Data = new System.IO.MemoryStream (new byte[] { 123 }); request.PartitionKey = "p1337"; return client.PutRecord (request); }
/// <summary> /// Initiates the asynchronous execution of the PutRecord operation. /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the PutRecord operation.</param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// <returns>The task object representing the asynchronous operation.</returns> public Task<PutRecordResponse> PutRecordAsync(PutRecordRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken)) { var marshaller = new PutRecordRequestMarshaller(); var unmarshaller = PutRecordResponseUnmarshaller.Instance; return InvokeAsync<PutRecordRequest,PutRecordResponse>(request, marshaller, unmarshaller, cancellationToken); }
internal PutRecordResponse PutRecord(PutRecordRequest request) { var marshaller = new PutRecordRequestMarshaller(); var unmarshaller = PutRecordResponseUnmarshaller.Instance; return Invoke<PutRecordRequest,PutRecordResponse>(request, marshaller, unmarshaller); }
/// <summary> /// Initiates the asynchronous execution of the PutRecord operation. /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the PutRecord operation on AmazonKinesisClient.</param> /// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param> /// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property.</param> /// /// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndPutRecord /// operation.</returns> public IAsyncResult BeginPutRecord(PutRecordRequest request, AsyncCallback callback, object state) { var marshaller = new PutRecordRequestMarshaller(); var unmarshaller = PutRecordResponseUnmarshaller.Instance; return BeginInvoke<PutRecordRequest>(request, marshaller, unmarshaller, callback, state); }
/// <summary> /// <para>This operation puts a data record into an Amazon Kinesis stream from a producer. This operation must be called to send data from the /// producer into the Amazon Kinesis stream for real-time ingestion and subsequent processing. The <c>PutRecord</c> operation requires the name /// of the stream that captures, stores, and transports the data; a partition key; and the data blob itself. The data blob could be a segment /// from a log file, geographic/location data, website clickstream data, or any other data type.</para> <para>The partition key is used to /// distribute data across shards. Amazon Kinesis segregates the data records that belong to a data stream into multiple shards, using the /// partition key associated with each data record to determine which shard a given data record belongs to. </para> <para>Partition keys are /// Unicode strings, with a maximum length limit of 256 bytes. An MD5 hash function is used to map partition keys to 128-bit integer values and /// to map associated data records to shards using the hash key ranges of the shards. You can override hashing the partition key to determine /// the shard by explicitly specifying a hash value using the <c>ExplicitHashKey</c> parameter. For more information, see the <a href="http://docs.aws.amazon.com/kinesis/latest/dev/">Amazon Kinesis Developer Guide</a> .</para> <para> <c>PutRecord</c> returns the shard /// ID of where the data record was placed and the sequence number that was assigned to the data record.</para> <para>Sequence numbers generally /// increase over time. To guarantee strictly increasing ordering, use the <c>SequenceNumberForOrdering</c> parameter. For more information, see /// the <a href="http://docs.aws.amazon.com/kinesis/latest/dev/">Amazon Kinesis Developer Guide</a> .</para> <para>If a <c>PutRecord</c> /// request cannot be processed because of insufficient provisioned throughput on the shard involved in the request, <c>PutRecord</c> throws /// <c>ProvisionedThroughputExceededException</c> . </para> <para>Data records are accessible for only 24 hours from the time that they are /// added to an Amazon Kinesis stream.</para> /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the PutRecord service method on AmazonKinesis.</param> /// /// <returns>The response from the PutRecord service method, as returned by AmazonKinesis.</returns> /// /// <exception cref="T:Amazon.Kinesis.Model.ProvisionedThroughputExceededException" /> /// <exception cref="T:Amazon.Kinesis.Model.InvalidArgumentException" /> /// <exception cref="T:Amazon.Kinesis.Model.ResourceNotFoundException" /> public PutRecordResponse PutRecord(PutRecordRequest request) { var task = PutRecordAsync(request); try { return task.Result; } catch(AggregateException e) { ExceptionDispatchInfo.Capture(e.InnerException).Throw(); return null; } }
/// <summary> /// Initiates the asynchronous execution of the PutRecord operation. /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the PutRecord operation on AmazonKinesisClient.</param> /// <param name="callback">An Action delegate that is invoked when the operation completes.</param> /// <param name="options">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property.</param> public void PutRecordAsync(PutRecordRequest request, AmazonServiceCallback<PutRecordRequest, PutRecordResponse> callback, AsyncOptions options = null) { options = options == null?new AsyncOptions():options; var marshaller = new PutRecordRequestMarshaller(); var unmarshaller = PutRecordResponseUnmarshaller.Instance; Action<AmazonWebServiceRequest, AmazonWebServiceResponse, Exception, AsyncOptions> callbackHelper = null; if(callback !=null ) callbackHelper = (AmazonWebServiceRequest req, AmazonWebServiceResponse res, Exception ex, AsyncOptions ao) => { AmazonServiceResult<PutRecordRequest,PutRecordResponse> responseObject = new AmazonServiceResult<PutRecordRequest,PutRecordResponse>((PutRecordRequest)req, (PutRecordResponse)res, ex , ao.State); callback(responseObject); }; BeginInvoke<PutRecordRequest>(request, marshaller, unmarshaller, options, callbackHelper); }
/// <summary> /// Initiates the asynchronous execution of the PutRecord operation. /// <seealso cref="Amazon.Kinesis.IAmazonKinesis.PutRecord"/> /// </summary> /// /// <param name="putRecordRequest">Container for the necessary parameters to execute the PutRecord operation on AmazonKinesis.</param> /// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param> /// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property.</param> /// /// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndPutRecord /// operation.</returns> public IAsyncResult BeginPutRecord(PutRecordRequest putRecordRequest, AsyncCallback callback, object state) { return invokePutRecord(putRecordRequest, callback, state, false); }
/// <summary> /// <para>This operation puts a data record into an Amazon Kinesis stream from a producer. This operation must be called to send data from the /// producer into the Amazon Kinesis stream for real-time ingestion and subsequent processing. The <c>PutRecord</c> operation requires the name /// of the stream that captures, stores, and transports the data; a partition key; and the data blob itself. The data blob could be a segment /// from a log file, geographic/location data, website clickstream data, or any other data type.</para> <para>The partition key is used to /// distribute data across shards. Amazon Kinesis segregates the data records that belong to a data stream into multiple shards, using the /// partition key associated with each data record to determine which shard a given data record belongs to. </para> <para>Partition keys are /// Unicode strings, with a maximum length limit of 256 bytes. An MD5 hash function is used to map partition keys to 128-bit integer values and /// to map associated data records to shards using the hash key ranges of the shards. You can override hashing the partition key to determine /// the shard by explicitly specifying a hash value using the <c>ExplicitHashKey</c> parameter. For more information, see the <a /// href="http://docs.aws.amazon.com/kinesis/latest/dev/" >Amazon Kinesis Developer Guide</a> .</para> <para> <c>PutRecord</c> returns the shard /// ID of where the data record was placed and the sequence number that was assigned to the data record.</para> <para>Sequence numbers generally /// increase over time. To guarantee strictly increasing ordering, use the <c>SequenceNumberForOrdering</c> parameter. For more information, see /// the <a href="http://docs.aws.amazon.com/kinesis/latest/dev/" >Amazon Kinesis Developer Guide</a> .</para> <para>If a <c>PutRecord</c> /// request cannot be processed because of insufficient provisioned throughput on the shard involved in the request, <c>PutRecord</c> throws /// <c>ProvisionedThroughputExceededException</c> . </para> <para>Data records are accessible for only 24 hours from the time that they are /// added to an Amazon Kinesis stream.</para> /// </summary> /// /// <param name="putRecordRequest">Container for the necessary parameters to execute the PutRecord service method on AmazonKinesis.</param> /// /// <returns>The response from the PutRecord service method, as returned by AmazonKinesis.</returns> /// /// <exception cref="ProvisionedThroughputExceededException"/> /// <exception cref="InvalidArgumentException"/> /// <exception cref="ResourceNotFoundException"/> public PutRecordResponse PutRecord(PutRecordRequest putRecordRequest) { IAsyncResult asyncResult = invokePutRecord(putRecordRequest, null, null, true); return EndPutRecord(asyncResult); }
private Amazon.Kinesis.Model.PutRecordResponse CallAWSServiceOperation(IAmazonKinesis client, Amazon.Kinesis.Model.PutRecordRequest request) { Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Kinesis", "PutRecord"); try { #if DESKTOP return(client.PutRecord(request)); #elif CORECLR return(client.PutRecordAsync(request).GetAwaiter().GetResult()); #else #error "Unknown build edition" #endif } catch (AmazonServiceException exc) { var webException = exc.InnerException as System.Net.WebException; if (webException != null) { throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException); } throw; } }
public object Execute(ExecutorContext context) { System.IO.MemoryStream _DataStream = null; try { var cmdletContext = context as CmdletContext; // create request var request = new Amazon.Kinesis.Model.PutRecordRequest(); if (cmdletContext.Data != null) { _DataStream = new System.IO.MemoryStream(cmdletContext.Data); request.Data = _DataStream; } if (cmdletContext.ExplicitHashKey != null) { request.ExplicitHashKey = cmdletContext.ExplicitHashKey; } if (cmdletContext.PartitionKey != null) { request.PartitionKey = cmdletContext.PartitionKey; } if (cmdletContext.SequenceNumberForOrdering != null) { request.SequenceNumberForOrdering = cmdletContext.SequenceNumberForOrdering; } if (cmdletContext.StreamName != null) { request.StreamName = cmdletContext.StreamName; } CmdletOutput output; // issue call var client = Client ?? CreateClient(_CurrentCredentials, _RegionEndpoint); try { var response = CallAWSServiceOperation(client, request); object pipelineOutput = null; pipelineOutput = cmdletContext.Select(response, this); output = new CmdletOutput { PipelineOutput = pipelineOutput, ServiceResponse = response }; } catch (Exception e) { output = new CmdletOutput { ErrorResponse = e }; } return(output); } finally { if (_DataStream != null) { _DataStream.Dispose(); } } }
internal PutRecordResponse PutRecord(PutRecordRequest request) { var task = PutRecordAsync(request); try { return task.Result; } catch(AggregateException e) { throw e.InnerException; } }
IAsyncResult invokePutRecord(PutRecordRequest putRecordRequest, AsyncCallback callback, object state, bool synchronized) { IRequest irequest = new PutRecordRequestMarshaller().Marshall(putRecordRequest); var unmarshaller = PutRecordResponseUnmarshaller.GetInstance(); AsyncResult result = new AsyncResult(irequest, callback, state, synchronized, signer, unmarshaller); Invoke(result); return result; }
/// <summary> /// <para>This operation puts a data record into an Amazon Kinesis stream from a producer. This operation must be called to send data from the /// producer into the Amazon Kinesis stream for real-time ingestion and subsequent processing. The <c>PutRecord</c> operation requires the name /// of the stream that captures, stores, and transports the data; a partition key; and the data blob itself. The data blob could be a segment /// from a log file, geographic/location data, website clickstream data, or any other data type.</para> <para>The partition key is used to /// distribute data across shards. Amazon Kinesis segregates the data records that belong to a data stream into multiple shards, using the /// partition key associated with each data record to determine which shard a given data record belongs to. </para> <para>Partition keys are /// Unicode strings, with a maximum length limit of 256 bytes. An MD5 hash function is used to map partition keys to 128-bit integer values and /// to map associated data records to shards using the hash key ranges of the shards. You can override hashing the partition key to determine /// the shard by explicitly specifying a hash value using the <c>ExplicitHashKey</c> parameter. For more information, see the <a href="http://docs.aws.amazon.com/kinesis/latest/dev/">Amazon Kinesis Developer Guide</a> .</para> <para> <c>PutRecord</c> returns the shard /// ID of where the data record was placed and the sequence number that was assigned to the data record.</para> <para>The /// <c>SequenceNumberForOrdering</c> sets the initial sequence number for the partition key. Later <c>PutRecord</c> requests to the same /// partition key (from the same client) will automatically increase from <c>SequenceNumberForOrdering</c> , ensuring strict sequential /// ordering.</para> <para>If a <c>PutRecord</c> request cannot be processed because of insufficient provisioned throughput on the shard /// involved in the request, <c>PutRecord</c> throws <c>ProvisionedThroughputExceededException</c> . </para> <para>Data records are accessible /// for only 24 hours from the time that they are added to an Amazon Kinesis stream.</para> /// </summary> /// /// <param name="putRecordRequest">Container for the necessary parameters to execute the PutRecord service method on AmazonKinesis.</param> /// /// <returns>The response from the PutRecord service method, as returned by AmazonKinesis.</returns> /// /// <exception cref="T:Amazon.Kinesis.Model.ProvisionedThroughputExceededException" /> /// <exception cref="T:Amazon.Kinesis.Model.InvalidArgumentException" /> /// <exception cref="T:Amazon.Kinesis.Model.ResourceNotFoundException" /> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> public Task<PutRecordResponse> PutRecordAsync(PutRecordRequest putRecordRequest, CancellationToken cancellationToken = default(CancellationToken)) { var marshaller = new PutRecordRequestMarshaller(); var unmarshaller = PutRecordResponseUnmarshaller.GetInstance(); return Invoke<IRequest, PutRecordRequest, PutRecordResponse>(putRecordRequest, marshaller, unmarshaller, signer, cancellationToken); }
static void Main(string[] args) { var o = new { Message = "Hello World", Author = "David Judd" }; //convert to byte array in prep for adding to stream byte[] oByte = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(o)); //create config that points to AWS region var config = new AmazonKinesisConfig(); config.RegionEndpoint = Amazon.RegionEndpoint.USEast1; //create client that pulls creds from web.config and takes in Kinesis config var client = new AmazonKinesisClient(config); var sw = Stopwatch.StartNew(); Stopwatch sw2 = null; var tasks = new List<Task<PutRecordResponse>>(); int count = int.Parse(ConfigurationManager.AppSettings["Count"]); Console.WriteLine("Sending {0} records... One at a time...", count); sw.Restart(); for (int i = 0; i < count; i++) { //System.Threading.Thread.Sleep(10); //sw2 = Stopwatch.StartNew(); //create stream object to add to Kinesis request using (MemoryStream ms = new MemoryStream(oByte)) { //create put request PutRecordRequest requestRecord = new PutRecordRequest(); //list name of Kinesis stream requestRecord.StreamName = "shomi_dev"; //give partition key that is used to place record in particular shard requestRecord.PartitionKey = i.ToString(); //add record as memorystream requestRecord.Data = ms; //PUT the record to Kinesis var task = client.PutRecordAsync(requestRecord); tasks.Add(task); } //sw2.Stop(); ///Console.WriteLine("Async latency is {0}", sw2.ElapsedMilliseconds); } Console.WriteLine("{0} records sent... Waiting for tasks to complete...", count); Task.WaitAll(tasks.ToArray(), -1); sw.Stop(); foreach (var t in tasks) { if (t.Result.HttpStatusCode != System.Net.HttpStatusCode.OK) { Console.WriteLine(t.Result.HttpStatusCode); } } double actionsPerSec = (double)count * 1000 / (double)sw.ElapsedMilliseconds; Console.WriteLine("{0} requests in {1} ms. {2:0.00} requests/sec.", count, sw.ElapsedMilliseconds, actionsPerSec); }