private static void ReadFromStream() { /*Config example*/ //AmazonKinesisConfig config = new AmazonKinesisConfig(); //config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1; //AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config); /*instance example*/ var kinesisClient = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1); String kinesisStreamName = "your-kinesis-name"; DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.StreamName = kinesisStreamName; DescribeStreamResponse describeResponse = kinesisClient.DescribeStream(describeRequest); List <Shard> shards = describeResponse.StreamDescription.Shards; foreach (Shard shard in shards) { GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.StreamName = kinesisStreamName; iteratorRequest.ShardId = shard.ShardId; iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON; GetShardIteratorResponse iteratorResponse = kinesisClient.GetShardIterator(iteratorRequest); string iteratorId = iteratorResponse.ShardIterator; while (!string.IsNullOrEmpty(iteratorId)) { GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.Limit = 1000; getRequest.ShardIterator = iteratorId; GetRecordsResponse getResponse = kinesisClient.GetRecords(getRequest); string nextIterator = getResponse.NextShardIterator; List <Record> records = getResponse.Records; if (records.Count > 0) { Console.WriteLine("Received {0} records. ", records.Count); foreach (Record record in records) { string json = Encoding.UTF8.GetString(record.Data.ToArray()); Console.WriteLine("Json string: " + json); } } iteratorId = nextIterator; } } }
private void UpdateData() { using (IAmazonKinesis klient = new AmazonKinesisClient(_credentials, Amazon.RegionEndpoint.USWest2)) { ListStreamsResponse resp = klient.ListStreams(); DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.StreamName = StreamName; DescribeStreamResponse describeResponse = klient.DescribeStream(describeRequest); List <Shard> shards = describeResponse.StreamDescription.Shards; foreach (Shard shard in shards) { SequenceNumberRange range = shard.SequenceNumberRange; GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.StreamName = StreamName; iteratorRequest.ShardId = shard.ShardId; iteratorRequest.ShardIteratorType = ShardIteratorType; GetShardIteratorResponse iteratorResponse = klient.GetShardIterator(iteratorRequest); string iteratorId = iteratorResponse.ShardIterator; while (IsRunning && !string.IsNullOrEmpty(iteratorId)) { Thread.Sleep(1); GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.Limit = 1000; getRequest.ShardIterator = iteratorId; GetRecordsResponse getResponse = klient.GetRecords(getRequest); string nextIterator = getResponse.NextShardIterator; List <Amazon.Kinesis.Model.Record> records = getResponse.Records; if (records.Count > 0) { if (IsDebug) { string message = string.Format("Received {0} records. ", records.Count); Console.WriteLine(message); foreach (IDebugObserver observer in _debugObservers) { observer.WriteDebug(message); } } foreach (Amazon.Kinesis.Model.Record record in records) { if (!IsRunning) { break; } Thread.Sleep(1); string json = Encoding.UTF8.GetString(record.Data.ToArray()); if (IsDebug) { string message = "Json string: " + json; Console.WriteLine(message); foreach (IDebugObserver observer in _debugObservers) { observer.WriteDebug(message); } } T obj = JsonConvert.DeserializeObject <T>(json); if (obj != null && _dataHandler != null) { _dataHandler(obj); } } } iteratorId = nextIterator; } if (!IsRunning) { break; } } } }
public List <JObject> GetRecords(Filter filter, string additionalFilterStr) { List <JObject> jsonObjects = new List <JObject>(); try { var response = _amazonKinesisClient.DescribeStream(new DescribeStreamRequest() { StreamName = _streamName }); foreach (var shard in response.StreamDescription.Shards) { var getRecordsRequest = new GetRecordsRequest(); getRecordsRequest.ShardIterator = _amazonKinesisClient.GetShardIterator(new GetShardIteratorRequest() { StreamName = _streamName, ShardIteratorType = ShardIteratorType.AT_TIMESTAMP, ShardId = shard.ShardId, Timestamp = _timePeriod }) .ShardIterator; getRecordsRequest.Limit = ConfigurationManager.AppSettings["FetchRecordsLimit"] != null?int.Parse(ConfigurationManager.AppSettings["FetchRecordsLimit"]) : 100; GetRecordsResponse recordResponse; dynamic additionalFilter = JObject.Parse(additionalFilterStr); do { recordResponse = _amazonKinesisClient.GetRecords(getRecordsRequest); getRecordsRequest.ShardIterator = recordResponse.NextShardIterator; foreach (var record in recordResponse.Records) { var sr = new StreamReader(record.Data); var myStr = sr.ReadToEnd(); dynamic jsonStr = JObject.Parse(myStr); bool isObjectQualified = FilterJsonObjectArray(filter, additionalFilter, jsonStr); if (isObjectQualified) { jsonObjects.Add(jsonStr); } } } while (recordResponse.MillisBehindLatest > 0); Console.Out.WriteLine("response = {0}", recordResponse); } } catch (Exception e) { Console.WriteLine(e); throw; } return(jsonObjects); }