Amazon Kinesis is a managed service that scales elastically for real time processing of streaming big data.
GetRecordsResponse GetRecords (AmazonKinesisClient client, string shardId) { var siRequest = new GetShardIteratorRequest (); siRequest.ShardId = shardId; siRequest.StreamName = "Test1"; siRequest.ShardIteratorType = "TRIM_HORIZON"; var siResponse = client.GetShardIterator (siRequest); var request = new GetRecordsRequest (); request.ShardIterator = siResponse.ShardIterator; return client.GetRecords (request); }
protected override void OnCreate (Bundle bundle) { base.OnCreate (bundle); // Set our view from the "main" layout resource SetContentView (Resource.Layout.Main); // Get our button from the layout resource, // and attach an event to it Button button = FindViewById<Button> (Resource.Id.myButton); button.Click += delegate { button.Text = string.Format ("{0} clicks!", count++); }; var client = new AmazonKinesisClient (new BasicAWSCredentials (ACCESS_KEY, SECRET_KEY), Amazon.RegionEndpoint.USEast1); var putResponse = PutRecord (client); var response = GetRecords (client, putResponse.ShardId); for (int i = 0; i < response.Records.Count; ++i) { Console.WriteLine ("Record: " + response.Records [i].Data.ReadByte ()); } }
public async Task<IHttpActionResult> Post() { var o = new { Message = "Hello World", Author = "David Judd" }; //convert to byte array in prep for adding to stream byte[] oByte = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(o)); //create client that pulls creds from web.config and takes in Kinesis config var client = new AmazonKinesisClient(Config); using (MemoryStream ms = new MemoryStream(oByte)) { //create put request PutRecordRequest requestRecord = new PutRecordRequest(); //list name of Kinesis stream requestRecord.StreamName = "shomi_dev"; //give partition key that is used to place record in particular shard requestRecord.PartitionKey = DateTime.Now.Ticks.ToString(); //add record as memorystream requestRecord.Data = ms; //PUT the record to Kinesis var response = await client.PutRecordAsync(requestRecord); return Ok(new { seq = response.SequenceNumber }); } }
PutRecordResponse PutRecord (AmazonKinesisClient client) { var request = new PutRecordRequest (); request.StreamName = "Test1"; request.Data = new System.IO.MemoryStream (new byte[] { 123 }); request.PartitionKey = "p1337"; return client.PutRecord (request); }
public static void Main() { SelfLog.Out = Console.Out; var client = new AmazonKinesisClient(); var streamOk = KinesisApi.CreateAndWaitForStreamToBecomeAvailable( kinesisClient: client, streamName: streamName, shardCount: shardCount ); var loggerConfig = new LoggerConfiguration() .WriteTo.ColoredConsole() .MinimumLevel.Debug(); if (streamOk) { loggerConfig.WriteTo.AmazonKinesis( kinesisClient: client, streamName: streamName, shardCount: shardCount, period: TimeSpan.FromSeconds(2), bufferBaseFilename: "./logs/kinesis-buffer", onLogSendError: OnLogSendError ); } Log.Logger = loggerConfig.CreateLogger(); #if false for (var i = 0; i < 50; i++) { for (int j = 0; j < 500; j++) { Thread.Sleep(1); Log.Debug("Count: {i} {j}", i, j); } Console.Write("."); } #endif LogStuff(); Log.Fatal("That's all folks - and all done using {WorkingSet} bytes of RAM", Environment.WorkingSet); Console.ReadKey(); }
public override void ViewDidLoad () { base.ViewDidLoad (); var client = new AmazonKinesisClient (new BasicAWSCredentials (ACCESS_KEY, SECRET_KEY), Amazon.RegionEndpoint.USEast1); var putResponse = PutRecord (client); var response = GetRecords (client, putResponse.ShardId); for (int i = 0; i < response.Records.Count; ++i) { Console.WriteLine ("Record: " + response.Records [i].Data.ReadByte ()); } }
static void Main(string[] args) { var o = new { Message = "Hello World", Author = "David Judd" }; //convert to byte array in prep for adding to stream byte[] oByte = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(o)); //create config that points to AWS region var config = new AmazonKinesisConfig(); config.RegionEndpoint = Amazon.RegionEndpoint.USEast1; //create client that pulls creds from web.config and takes in Kinesis config var client = new AmazonKinesisClient(config); var sw = Stopwatch.StartNew(); Stopwatch sw2 = null; var tasks = new List<Task<PutRecordResponse>>(); int count = int.Parse(ConfigurationManager.AppSettings["Count"]); Console.WriteLine("Sending {0} records... One at a time...", count); sw.Restart(); for (int i = 0; i < count; i++) { //System.Threading.Thread.Sleep(10); //sw2 = Stopwatch.StartNew(); //create stream object to add to Kinesis request using (MemoryStream ms = new MemoryStream(oByte)) { //create put request PutRecordRequest requestRecord = new PutRecordRequest(); //list name of Kinesis stream requestRecord.StreamName = "shomi_dev"; //give partition key that is used to place record in particular shard requestRecord.PartitionKey = i.ToString(); //add record as memorystream requestRecord.Data = ms; //PUT the record to Kinesis var task = client.PutRecordAsync(requestRecord); tasks.Add(task); } //sw2.Stop(); ///Console.WriteLine("Async latency is {0}", sw2.ElapsedMilliseconds); } Console.WriteLine("{0} records sent... Waiting for tasks to complete...", count); Task.WaitAll(tasks.ToArray(), -1); sw.Stop(); foreach (var t in tasks) { if (t.Result.HttpStatusCode != System.Net.HttpStatusCode.OK) { Console.WriteLine(t.Result.HttpStatusCode); } } double actionsPerSec = (double)count * 1000 / (double)sw.ElapsedMilliseconds; Console.WriteLine("{0} requests in {1} ms. {2:0.00} requests/sec.", count, sw.ElapsedMilliseconds, actionsPerSec); }