public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems) { AmazonKinesisConfig config = new AmazonKinesisConfig(); config.RegionEndpoint = region; ConfigureClient(config); AmazonKinesisClient client = new AmazonKinesisClient(creds, config); DescribeStreamResponse resp = new DescribeStreamResponse(); do { DescribeStreamRequest req = new DescribeStreamRequest { ExclusiveStartShardId = resp.StreamDescription.Shards[0].ShardId , Limit = maxItems }; resp = client.DescribeStream(req); CheckError(resp.HttpStatusCode, "200"); foreach (var obj in resp.StreamDescription.Shards) { AddObject(obj); } }while (!string.IsNullOrEmpty(resp.StreamDescription.Shards[0].ShardId)); }
private static void ReadFromStream() { /*Config example*/ //AmazonKinesisConfig config = new AmazonKinesisConfig(); //config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1; //AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config); /*instance example*/ var kinesisClient = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1); String kinesisStreamName = "your-kinesis-name"; DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.StreamName = kinesisStreamName; DescribeStreamResponse describeResponse = kinesisClient.DescribeStream(describeRequest); List <Shard> shards = describeResponse.StreamDescription.Shards; foreach (Shard shard in shards) { GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.StreamName = kinesisStreamName; iteratorRequest.ShardId = shard.ShardId; iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON; GetShardIteratorResponse iteratorResponse = kinesisClient.GetShardIterator(iteratorRequest); string iteratorId = iteratorResponse.ShardIterator; while (!string.IsNullOrEmpty(iteratorId)) { GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.Limit = 1000; getRequest.ShardIterator = iteratorId; GetRecordsResponse getResponse = kinesisClient.GetRecords(getRequest); string nextIterator = getResponse.NextShardIterator; List <Record> records = getResponse.Records; if (records.Count > 0) { Console.WriteLine("Received {0} records. ", records.Count); foreach (Record record in records) { string json = Encoding.UTF8.GetString(record.Data.ToArray()); Console.WriteLine("Json string: " + json); } } iteratorId = nextIterator; } } }
public DescribeStreamResponse GetStreamResponse(string streamName = null) { if (string.IsNullOrEmpty(streamName) && string.IsNullOrEmpty(_streamName)) { throw new Exception("Please specify a stream name to get the stream response."); } var request = new DescribeStreamRequest() { StreamName = streamName ?? _streamName }; return(_client.DescribeStream(request)); }
/// <summary> /// This method waits a maximum of 10 minutes for the specified stream to become active. /// <param name="myStreamName">Name of the stream whose active status is waited upon.</param> /// </summary> private static void WaitForStreamToBecomeAvailable(string myStreamName) { var deadline = DateTime.UtcNow + TimeSpan.FromMinutes(10); while (DateTime.UtcNow < deadline) { DescribeStreamRequest describeStreamReq = new DescribeStreamRequest(); describeStreamReq.StreamName = myStreamName; DescribeStreamResult describeResult = kinesisClient.DescribeStream(describeStreamReq); string streamStatus = describeResult.StreamDescription.StreamStatus; Console.Error.WriteLine(" - current state: " + streamStatus); if (streamStatus == StreamStatus.ACTIVE) { return; } Thread.Sleep(TimeSpan.FromSeconds(20)); } throw new Exception("Stream " + myStreamName + " never went active."); }
private void UpdateData() { using (IAmazonKinesis klient = new AmazonKinesisClient(_credentials, Amazon.RegionEndpoint.USWest2)) { ListStreamsResponse resp = klient.ListStreams(); DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.StreamName = StreamName; DescribeStreamResponse describeResponse = klient.DescribeStream(describeRequest); List <Shard> shards = describeResponse.StreamDescription.Shards; foreach (Shard shard in shards) { SequenceNumberRange range = shard.SequenceNumberRange; GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.StreamName = StreamName; iteratorRequest.ShardId = shard.ShardId; iteratorRequest.ShardIteratorType = ShardIteratorType; GetShardIteratorResponse iteratorResponse = klient.GetShardIterator(iteratorRequest); string iteratorId = iteratorResponse.ShardIterator; while (IsRunning && !string.IsNullOrEmpty(iteratorId)) { Thread.Sleep(1); GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.Limit = 1000; getRequest.ShardIterator = iteratorId; GetRecordsResponse getResponse = klient.GetRecords(getRequest); string nextIterator = getResponse.NextShardIterator; List <Amazon.Kinesis.Model.Record> records = getResponse.Records; if (records.Count > 0) { if (IsDebug) { string message = string.Format("Received {0} records. ", records.Count); Console.WriteLine(message); foreach (IDebugObserver observer in _debugObservers) { observer.WriteDebug(message); } } foreach (Amazon.Kinesis.Model.Record record in records) { if (!IsRunning) { break; } Thread.Sleep(1); string json = Encoding.UTF8.GetString(record.Data.ToArray()); if (IsDebug) { string message = "Json string: " + json; Console.WriteLine(message); foreach (IDebugObserver observer in _debugObservers) { observer.WriteDebug(message); } } T obj = JsonConvert.DeserializeObject <T>(json); if (obj != null && _dataHandler != null) { _dataHandler(obj); } } } iteratorId = nextIterator; } if (!IsRunning) { break; } } } }
public List <JObject> GetRecords(Filter filter, string additionalFilterStr) { List <JObject> jsonObjects = new List <JObject>(); try { var response = _amazonKinesisClient.DescribeStream(new DescribeStreamRequest() { StreamName = _streamName }); foreach (var shard in response.StreamDescription.Shards) { var getRecordsRequest = new GetRecordsRequest(); getRecordsRequest.ShardIterator = _amazonKinesisClient.GetShardIterator(new GetShardIteratorRequest() { StreamName = _streamName, ShardIteratorType = ShardIteratorType.AT_TIMESTAMP, ShardId = shard.ShardId, Timestamp = _timePeriod }) .ShardIterator; getRecordsRequest.Limit = ConfigurationManager.AppSettings["FetchRecordsLimit"] != null?int.Parse(ConfigurationManager.AppSettings["FetchRecordsLimit"]) : 100; GetRecordsResponse recordResponse; dynamic additionalFilter = JObject.Parse(additionalFilterStr); do { recordResponse = _amazonKinesisClient.GetRecords(getRecordsRequest); getRecordsRequest.ShardIterator = recordResponse.NextShardIterator; foreach (var record in recordResponse.Records) { var sr = new StreamReader(record.Data); var myStr = sr.ReadToEnd(); dynamic jsonStr = JObject.Parse(myStr); bool isObjectQualified = FilterJsonObjectArray(filter, additionalFilter, jsonStr); if (isObjectQualified) { jsonObjects.Add(jsonStr); } } } while (recordResponse.MillisBehindLatest > 0); Console.Out.WriteLine("response = {0}", recordResponse); } } catch (Exception e) { Console.WriteLine(e); throw; } return(jsonObjects); }