public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context) { GetRecordsResponse response = new GetRecordsResponse(); context.Read(); int targetDepth = context.CurrentDepth; while (context.ReadAtDepth(targetDepth)) { if (context.TestExpression("Records", targetDepth)) { var unmarshaller = new ListUnmarshaller <Record, RecordUnmarshaller>( RecordUnmarshaller.GetInstance()); response.Records = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("NextShardIterator", targetDepth)) { response.NextShardIterator = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } } return(response); }
private async Task <string> TryProcessRecordsAsync(Checkpoint checkPoint, string iterator, CancellationToken token) { GetRecordsRequest request = new GetRecordsRequest { Limit = 250, ShardIterator = iterator, }; GetRecordsResponse response = await _client.GetRecordsAsync(request, token); UpdateHeartbeatDelay(response.MillisBehindLatest); if (response.Records.Any()) { ProcessRecordsAsync(response, checkPoint).Wait(); CheckpointRepository.Save(checkPoint); } if (response.Records.Count < 10) { Trace.WriteLine("Sleeping as no new data."); Task.Delay(1000, token).Wait(token); } else { // _logger.LogInformation("Got {0} records from Kinesis", response.Records.Count); } iterator = response.NextShardIterator; return(iterator); }
public static RecordResponse Create(GetRecordsResponse getRecordsResponse, CancellationToken cancellationToken) { return(new RecordResponse() { GetRecordsResponse = getRecordsResponse, CancellationToken = cancellationToken }); }
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context) { GetRecordsResponse response = new GetRecordsResponse(); context.Read(); UnmarshallResult(context, response); return(response); }
public void UpdateShardInformation(GetRecordsResponse recordsResponse) { SetNextShardIterator(recordsResponse.NextShardIterator); if (recordsResponse.Records.Count > 0) { SequenceNumber = recordsResponse.Records.LastOrDefault().SequenceNumber; } LastUpdateUtc = DateTime.UtcNow; }
private static void ReadFromStream() { /*Config example*/ //AmazonKinesisConfig config = new AmazonKinesisConfig(); //config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1; //AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config); /*instance example*/ var kinesisClient = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1); String kinesisStreamName = "your-kinesis-name"; DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.StreamName = kinesisStreamName; DescribeStreamResponse describeResponse = kinesisClient.DescribeStream(describeRequest); List <Shard> shards = describeResponse.StreamDescription.Shards; foreach (Shard shard in shards) { GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.StreamName = kinesisStreamName; iteratorRequest.ShardId = shard.ShardId; iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON; GetShardIteratorResponse iteratorResponse = kinesisClient.GetShardIterator(iteratorRequest); string iteratorId = iteratorResponse.ShardIterator; while (!string.IsNullOrEmpty(iteratorId)) { GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.Limit = 1000; getRequest.ShardIterator = iteratorId; GetRecordsResponse getResponse = kinesisClient.GetRecords(getRequest); string nextIterator = getResponse.NextShardIterator; List <Record> records = getResponse.Records; if (records.Count > 0) { Console.WriteLine("Received {0} records. ", records.Count); foreach (Record record in records) { string json = Encoding.UTF8.GetString(record.Data.ToArray()); Console.WriteLine("Json string: " + json); } } iteratorId = nextIterator; } } }
private async Task ReadFromStream() { DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.StreamName = myStreamName; DescribeStreamResponse describeResponse = await kinesisClient.DescribeStreamAsync(describeRequest); List <Shard> shards = describeResponse.StreamDescription.Shards; foreach (Shard shard in shards) { GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.StreamName = myStreamName; iteratorRequest.ShardId = shard.ShardId; iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON; GetShardIteratorResponse iteratorResponse = await kinesisClient.GetShardIteratorAsync(iteratorRequest); string iteratorId = iteratorResponse.ShardIterator; while (!string.IsNullOrEmpty(iteratorId)) { GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.Limit = 1000; getRequest.ShardIterator = iteratorId; GetRecordsResponse getResponse = await kinesisClient.GetRecordsAsync(getRequest); string nextIterator = getResponse.NextShardIterator; List <Record> records = getResponse.Records; if (records.Count > 0) { Console.WriteLine("Received {0} records. ", records.Count); foreach (Record record in records) { string theMessage = Encoding.UTF8.GetString(record.Data.ToArray()); Console.WriteLine("message string: " + theMessage); } } iteratorId = nextIterator; } } }
private static void UnmarshallResult(JsonUnmarshallerContext context, GetRecordsResponse response) { int originalDepth = context.CurrentDepth; int targetDepth = originalDepth + 1; while (context.Read()) { if (context.TestExpression("Records", targetDepth)) { context.Read(); response.Records = new List <Record>(); RecordUnmarshaller unmarshaller = RecordUnmarshaller.GetInstance(); while (context.Read()) { JsonToken token = context.CurrentTokenType; if (token == JsonToken.ArrayStart) { continue; } if (token == JsonToken.ArrayEnd) { break; } response.Records.Add(unmarshaller.Unmarshall(context)); } continue; } if (context.TestExpression("NextShardIterator", targetDepth)) { context.Read(); response.NextShardIterator = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.CurrentDepth <= originalDepth) { return; } } return; }
public GetRecordsSuccess(GetRecordsResponse response) { Response = response; }
private async Task ReadStream() { List <Models.RefPerson> refPersons = new List <Models.RefPerson>(); dataGridDetectedPersons.ItemsSource = refPersons; dataGridDetectedPersons.Items.Refresh(); string streamArn = "arn:aws:dynamodb:ap-southeast-2:358403828169:table/ref_persons/stream/2019-11-18T05:31:40.045"; int maxItemCount = 100; try { AmazonDynamoDBStreamsClient streamsClient; using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.dynamodbRegion)) { String lastEvaluatedShardId = null; do { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest() { StreamArn = streamArn, ExclusiveStartShardId = lastEvaluatedShardId, }; DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest); List <Shard> shards = describeStreamResponse.StreamDescription.Shards; // Process each shard on this page foreach (Shard shard in shards) { String shardId = shard.ShardId; // Get an iterator for the current shard GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest() { StreamArn = streamArn, ShardId = shardId, ShardIteratorType = ShardIteratorType.LATEST, }; GetShardIteratorResponse getShardIteratorResponse = await streamsClient.GetShardIteratorAsync(getShardIteratorRequest); String currentShardIter = getShardIteratorResponse.ShardIterator; int processedRecordCount = 0; while (currentShardIter != null && processedRecordCount < maxItemCount) { // Use the shard iterator to read the stream records GetRecordsRequest getRecordsRequest = new GetRecordsRequest() { ShardIterator = currentShardIter }; GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest); List <Record> records = getRecordsResponse.Records; foreach (Record record in records) { foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage) { string changedRefPersonId = record.Dynamodb.NewImage["id"].S; string changedRefPersonStatus = record.Dynamodb.NewImage["status"].N.ToString(); string changedRefPersonName = record.Dynamodb.NewImage["name"].S; string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S; //Console.WriteLine($"{changedRefPersonId}:{changedRefPersonStatus}:{changedRefPersonName}:{changedRefPersonDescription}"); Models.RefPerson refPerson = new Models.RefPerson(); refPerson.id = changedRefPersonId; refPerson.name = changedRefPersonName; refPerson.status = (changedRefPersonStatus == "1")?true:false; refPerson.description = changedRefPersonDescription; refPerson.camera = "mobile_stream_1"; string directoryPath = "Resources/Images/"; if (!File.Exists(directoryPath + refPerson.id)) { Models.S3Bucket.DownloadFile(refPerson.id); } string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\"; Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id); refPerson.image = new BitmapImage(fileUri); if (refPerson.status) { if (refPersons.FindAll(p => p.id == refPerson.id).Count == 0) { refPersons.Add(refPerson); } } else { refPersons.RemoveAll(p => p.id == refPerson.id); } dataGridDetectedPersons.ItemsSource = refPersons; dataGridDetectedPersons.Items.Refresh(); } } processedRecordCount += records.Count; currentShardIter = getRecordsResponse.NextShardIterator; } } // If LastEvaluatedShardId is set, then there is // at least one more page of shard IDs to retrieve lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId; } while (lastEvaluatedShardId != null); } } catch (AmazonDynamoDBException e) { Console.WriteLine("AmazonDynamoDBException: " + e); } catch (Exception e) { Console.WriteLine("Error: " + e); } }
private void UpdateData() { using (IAmazonKinesis klient = new AmazonKinesisClient(_credentials, Amazon.RegionEndpoint.USWest2)) { ListStreamsResponse resp = klient.ListStreams(); DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.StreamName = StreamName; DescribeStreamResponse describeResponse = klient.DescribeStream(describeRequest); List <Shard> shards = describeResponse.StreamDescription.Shards; foreach (Shard shard in shards) { SequenceNumberRange range = shard.SequenceNumberRange; GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.StreamName = StreamName; iteratorRequest.ShardId = shard.ShardId; iteratorRequest.ShardIteratorType = ShardIteratorType; GetShardIteratorResponse iteratorResponse = klient.GetShardIterator(iteratorRequest); string iteratorId = iteratorResponse.ShardIterator; while (IsRunning && !string.IsNullOrEmpty(iteratorId)) { Thread.Sleep(1); GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.Limit = 1000; getRequest.ShardIterator = iteratorId; GetRecordsResponse getResponse = klient.GetRecords(getRequest); string nextIterator = getResponse.NextShardIterator; List <Amazon.Kinesis.Model.Record> records = getResponse.Records; if (records.Count > 0) { if (IsDebug) { string message = string.Format("Received {0} records. ", records.Count); Console.WriteLine(message); foreach (IDebugObserver observer in _debugObservers) { observer.WriteDebug(message); } } foreach (Amazon.Kinesis.Model.Record record in records) { if (!IsRunning) { break; } Thread.Sleep(1); string json = Encoding.UTF8.GetString(record.Data.ToArray()); if (IsDebug) { string message = "Json string: " + json; Console.WriteLine(message); foreach (IDebugObserver observer in _debugObservers) { observer.WriteDebug(message); } } T obj = JsonConvert.DeserializeObject <T>(json); if (obj != null && _dataHandler != null) { _dataHandler(obj); } } } iteratorId = nextIterator; } if (!IsRunning) { break; } } } }
private async Task ReadRefPersonStream() { Dictionary <string, Models.RefPerson> refPersons = new Dictionary <string, Models.RefPerson>(); refPersons.Clear(); dataGridDetectedPersons.ItemsSource = refPersons.Values; dataGridDetectedPersons.Items.Refresh(); string streamArn = Models.MyAWSConfigs.DynamodbRefPersonTableStreamArn; //int maxItemCount = 100; try { AmazonDynamoDBStreamsClient streamsClient; using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion)) { String lastEvaluatedShardId = null; do { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest() { StreamArn = streamArn, ExclusiveStartShardId = lastEvaluatedShardId, }; DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest); List <Shard> shards = describeStreamResponse.StreamDescription.Shards; // Process each shard on this page foreach (Shard shard in shards) { String shardId = shard.ShardId; // Get an iterator for the current shard GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest() { StreamArn = streamArn, ShardId = shardId, ShardIteratorType = ShardIteratorType.LATEST, }; GetShardIteratorResponse getShardIteratorResponse = await streamsClient.GetShardIteratorAsync(getShardIteratorRequest); String currentShardIter = getShardIteratorResponse.ShardIterator; int processedRecordCount = 0; //&& processedRecordCount < maxItemCount while (currentShardIter != null) { // Use the shard iterator to read the stream records GetRecordsRequest getRecordsRequest = new GetRecordsRequest() { ShardIterator = currentShardIter }; GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest); List <Record> records = getRecordsResponse.Records; foreach (Record record in records) { foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage) { string changedRefPersonId = record.Dynamodb.NewImage["id"].S; string changedRefPersonStatus = record.Dynamodb.NewImage["status"].N.ToString(); string changedRefPersonName = record.Dynamodb.NewImage["name"].S; string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S; string changedRefPersonCamera = record.Dynamodb.NewImage["camera"].S; Models.RefPerson refPerson = new Models.RefPerson(); refPerson.id = changedRefPersonId; refPerson.name = changedRefPersonName; refPerson.status = (changedRefPersonStatus == "1") ? true : false; refPerson.description = changedRefPersonDescription; refPerson.camera = changedRefPersonCamera; refPerson.lastLocation = cameras[changedRefPersonCamera].location; string directoryPath = "Resources/Images/"; if (!File.Exists(directoryPath + refPerson.id)) { Models.S3Bucket.DownloadFile(refPerson.id, Models.MyAWSConfigs.RefImagesBucketName); } string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\"; Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id); refPerson.image = new BitmapImage(fileUri); if (refPerson.status) { if (!refPersons.ContainsKey(refPerson.id)) { refPersons.Add(refPerson.id, refPerson); } } else { if (refPersons.ContainsKey(refPerson.id)) { refPersons.Remove(refPerson.id); } } dataGridDetectedPersons.ItemsSource = refPersons.Values; dataGridDetectedPersons.Items.Refresh(); } } processedRecordCount += records.Count; currentShardIter = getRecordsResponse.NextShardIterator; } } // If LastEvaluatedShardId is set, then there is // at least one more page of shard IDs to retrieve lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId; } while (lastEvaluatedShardId != null); } } catch (AmazonDynamoDBException e) { Console.WriteLine("AmazonDynamoDBException: " + e); } catch (Exception e) { Console.WriteLine("Error: " + e); } }
private async Task ReadHistoryStream() { string streamArn = Models.MyAWSConfigs.DynamodbHistoryTableStreamArn; DateTime notifyTime = DateTime.Now; notifyTime = DateTime.Now.AddMilliseconds(notifyInterval); //int maxItemCount = 100; try { AmazonDynamoDBStreamsClient streamsClient; using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion)) { String lastEvaluatedShardId = null; do { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest() { StreamArn = streamArn, ExclusiveStartShardId = lastEvaluatedShardId, }; DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest); List <Shard> shards = describeStreamResponse.StreamDescription.Shards; // Process each shard on this page foreach (Shard shard in shards) { String shardId = shard.ShardId; // Get an iterator for the current shard GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest() { StreamArn = streamArn, ShardId = shardId, ShardIteratorType = ShardIteratorType.LATEST, }; GetShardIteratorResponse getShardIteratorResponse = await streamsClient.GetShardIteratorAsync(getShardIteratorRequest); String currentShardIter = getShardIteratorResponse.ShardIterator; int processedRecordCount = 0; //&& processedRecordCount < maxItemCount while (currentShardIter != null) { // Use the shard iterator to read the stream records GetRecordsRequest getRecordsRequest = new GetRecordsRequest() { ShardIterator = currentShardIter }; GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest); List <Record> records = getRecordsResponse.Records; foreach (Record record in records) { foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage) { if (DateTime.Now >= notifyTime) { string id = record.Dynamodb.NewImage["id"].S; string name = record.Dynamodb.NewImage["name"].S; string camId = record.Dynamodb.NewImage["cameraId"].S; string camLocation = cameras[camId].location; string notification = $"{name} has been detected by camera {camId} at {camLocation}"; string trayMessage = $"{name} has been detected"; AddNewNotification(notification, id).Wait(); notifyTime = DateTime.Now.AddMilliseconds(notifyInterval); notifyIcon.Visible = true; notifyIcon.ShowBalloonTip(1000, "New Person Detected", trayMessage, System.Windows.Forms.ToolTipIcon.Info); } } } processedRecordCount += records.Count; currentShardIter = getRecordsResponse.NextShardIterator; } } // If LastEvaluatedShardId is set, then there is // at least one more page of shard IDs to retrieve lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId; } while (lastEvaluatedShardId != null); } } catch (AmazonDynamoDBException e) { Console.WriteLine("AmazonDynamoDBException: " + e); } catch (Exception e) { Console.WriteLine("Error: " + e); } }
protected override async Task ProcessRecordsAsync(GetRecordsResponse recordsResponse, Checkpoint checkPoint) { await ProcessMeasurements(recordsResponse.Records, checkPoint); }
protected abstract Task ProcessRecordsAsync(GetRecordsResponse recordsResponse, Checkpoint checkPoint);
public IActionResult GetBatch(GetRecordsRequest request) { var getResponse = new GetRecordsResponse(); return(Ok(getResponse)); }