public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems)
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = region;
ConfigureClient(config);
AmazonKinesisClient client = new AmazonKinesisClient(creds, config);
DescribeStreamResponse resp = new DescribeStreamResponse();
do
{
DescribeStreamRequest req = new DescribeStreamRequest
{
ExclusiveStartShardId = resp.StreamDescription.Shards[0].ShardId
,
Limit = maxItems
};
resp = client.DescribeStream(req);
CheckError(resp.HttpStatusCode, "200");
foreach (var obj in resp.StreamDescription.Shards)
{
AddObject(obj);
}
}while (!string.IsNullOrEmpty(resp.StreamDescription.Shards[0].ShardId));
}
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
DescribeStreamResponse response = new DescribeStreamResponse();
context.Read();
UnmarshallResult(context, response);
return(response);
}
private static void ReadFromStream()
{
/*Config example*/
//AmazonKinesisConfig config = new AmazonKinesisConfig();
//config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1;
//AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
/*instance example*/
var kinesisClient = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1);
String kinesisStreamName = "your-kinesis-name";
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = kinesisStreamName;
DescribeStreamResponse describeResponse = kinesisClient.DescribeStream(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = kinesisStreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON;
GetShardIteratorResponse iteratorResponse = kinesisClient.GetShardIterator(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(iteratorId))
{
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = kinesisClient.GetRecords(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Record> records = getResponse.Records;
if (records.Count > 0)
{
Console.WriteLine("Received {0} records. ", records.Count);
foreach (Record record in records)
{
string json = Encoding.UTF8.GetString(record.Data.ToArray());
Console.WriteLine("Json string: " + json);
}
}
iteratorId = nextIterator;
}
}
}
private async Task ReadFromStream()
{
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = myStreamName;
DescribeStreamResponse describeResponse =
await kinesisClient.DescribeStreamAsync(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = myStreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON;
GetShardIteratorResponse iteratorResponse = await kinesisClient.GetShardIteratorAsync(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(iteratorId))
{
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = await kinesisClient.GetRecordsAsync(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Record> records = getResponse.Records;
if (records.Count > 0)
{
Console.WriteLine("Received {0} records. ", records.Count);
foreach (Record record in records)
{
string theMessage = Encoding.UTF8.GetString(record.Data.ToArray());
Console.WriteLine("message string: " + theMessage);
}
}
iteratorId = nextIterator;
}
}
}
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
DescribeStreamResponse response = new DescribeStreamResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.TestExpression("StreamDescription", targetDepth))
{
response.StreamDescription = StreamDescriptionUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
}
return(response);
}
/// <summary>
/// Called when the Kinesis Stream Awaiter is created in the CF script, it will wait on the specified stream to enter
/// ACTIVE status
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task <CustomResourceResponse> CreateAsync(CustomResourceRequest request, ILambdaContext context)
{
if (request.ResourceProperties.ContainsKey("StreamName"))
{
context.LogInfo($"Beginning await for Kinesis stream {request.ResourceProperties["StreamName"]}.");
DescribeStreamRequest Request = new DescribeStreamRequest()
{
StreamName = request.ResourceProperties["StreamName"].ToString()
};
while (true)
{
if (context.RemainingTime.TotalMilliseconds < 1500)
{
return(new CustomResourceResponse(CustomResourceResponse.RequestStatus.FAILED, "Timeout waiting for stream to become active.", request));
}
DescribeStreamResponse Response = await this._KinesisClient.DescribeStreamAsync(Request);
if ((int)Response.HttpStatusCode < 300)
{
if (Response.StreamDescription.StreamStatus == StreamStatus.ACTIVE)
{
break;
}
}
else
{
context.LogWarning($"Received an unsuccessful response to the describe stream request: {(int)Response.HttpStatusCode}.");
}
Thread.Sleep(_WaitTimeInMillis);
}
context.LogInfo($"Successfully created Kinesis stream {Request.StreamName}.");
return(new CustomResourceResponse(CustomResourceResponse.RequestStatus.SUCCESS, "Created", Request.StreamName, request.StackId, request.RequestId, request.LogicalResourceId));
}
else
{
return(new CustomResourceResponse(CustomResourceResponse.RequestStatus.FAILED, "The StreamName property was not provided.", "stream", request.StackId, request.RequestId, request.LogicalResourceId));
}
}
/// <summary>
/// This method waits a maximum of 10 minutes for the specified stream to become active.
/// <param name="myStreamName">Name of the stream whose active status is waited upon.</param>
/// </summary>
private static bool WaitForStreamToBecomeAvailable(string myStreamName)
{
var deadline = DateTime.UtcNow + TimeSpan.FromMinutes(10);
while (DateTime.UtcNow < deadline)
{
DescribeStreamRequest describeStreamReq = new DescribeStreamRequest();
describeStreamReq.StreamName = myStreamName;
DescribeStreamResponse describeResult = oClient.DescribeStream(describeStreamReq);
string streamStatus = describeResult.HttpStatusCode.ToString();
Console.Error.WriteLine(" - current state: " + streamStatus);
if (streamStatus == StreamStatus.ACTIVE)
{
return(true);
}
Thread.Sleep(TimeSpan.FromSeconds(20));
}
throw new Exception("Stream " + myStreamName + " never went active.");
}
private static void UnmarshallResult(JsonUnmarshallerContext context, DescribeStreamResponse response)
{
int originalDepth = context.CurrentDepth;
int targetDepth = originalDepth + 1;
while (context.Read())
{
if (context.TestExpression("StreamDescription", targetDepth))
{
context.Read();
response.StreamDescription = StreamDescriptionUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.CurrentDepth <= originalDepth)
{
return;
}
}
return;
}
public static DescribeStreamResponse Unmarshall(UnmarshallerContext context)
{
DescribeStreamResponse describeStreamResponse = new DescribeStreamResponse();
describeStreamResponse.HttpResponse = context.HttpResponse;
describeStreamResponse.RequestId = context.StringValue("DescribeStream.RequestId");
describeStreamResponse.Id = context.StringValue("DescribeStream.Id");
describeStreamResponse.Name = context.StringValue("DescribeStream.Name");
describeStreamResponse.PushDomain = context.StringValue("DescribeStream.PushDomain");
describeStreamResponse.PlayDomain = context.StringValue("DescribeStream.PlayDomain");
describeStreamResponse.App = context.StringValue("DescribeStream.App");
describeStreamResponse.Protocol = context.StringValue("DescribeStream.Protocol");
describeStreamResponse.GroupId = context.StringValue("DescribeStream.GroupId");
describeStreamResponse.DeviceId = context.StringValue("DescribeStream.DeviceId");
describeStreamResponse.Enabled = context.BooleanValue("DescribeStream.Enabled");
describeStreamResponse.Status = context.StringValue("DescribeStream.Status");
describeStreamResponse.Height = context.IntegerValue("DescribeStream.Height");
describeStreamResponse.Width = context.IntegerValue("DescribeStream.Width");
describeStreamResponse.CreatedTime = context.StringValue("DescribeStream.CreatedTime");
return(describeStreamResponse);
}
private async Task ReadStream()
{
List <Models.RefPerson> refPersons = new List <Models.RefPerson>();
dataGridDetectedPersons.ItemsSource = refPersons;
dataGridDetectedPersons.Items.Refresh();
string streamArn = "arn:aws:dynamodb:ap-southeast-2:358403828169:table/ref_persons/stream/2019-11-18T05:31:40.045";
int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.dynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
while (currentShardIter != null && processedRecordCount < maxItemCount)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
string changedRefPersonId = record.Dynamodb.NewImage["id"].S;
string changedRefPersonStatus = record.Dynamodb.NewImage["status"].N.ToString();
string changedRefPersonName = record.Dynamodb.NewImage["name"].S;
string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S;
//Console.WriteLine($"{changedRefPersonId}:{changedRefPersonStatus}:{changedRefPersonName}:{changedRefPersonDescription}");
Models.RefPerson refPerson = new Models.RefPerson();
refPerson.id = changedRefPersonId;
refPerson.name = changedRefPersonName;
refPerson.status = (changedRefPersonStatus == "1")?true:false;
refPerson.description = changedRefPersonDescription;
refPerson.camera = "mobile_stream_1";
string directoryPath = "Resources/Images/";
if (!File.Exists(directoryPath + refPerson.id))
{
Models.S3Bucket.DownloadFile(refPerson.id);
}
string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\";
Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id);
refPerson.image = new BitmapImage(fileUri);
if (refPerson.status)
{
if (refPersons.FindAll(p => p.id == refPerson.id).Count == 0)
{
refPersons.Add(refPerson);
}
}
else
{
refPersons.RemoveAll(p => p.id == refPerson.id);
}
dataGridDetectedPersons.ItemsSource = refPersons;
dataGridDetectedPersons.Items.Refresh();
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
private void UpdateData()
{
using (IAmazonKinesis klient = new AmazonKinesisClient(_credentials, Amazon.RegionEndpoint.USWest2))
{
ListStreamsResponse resp = klient.ListStreams();
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = StreamName;
DescribeStreamResponse describeResponse = klient.DescribeStream(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
SequenceNumberRange range = shard.SequenceNumberRange;
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = StreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType;
GetShardIteratorResponse iteratorResponse = klient.GetShardIterator(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (IsRunning && !string.IsNullOrEmpty(iteratorId))
{
Thread.Sleep(1);
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = klient.GetRecords(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Amazon.Kinesis.Model.Record> records = getResponse.Records;
if (records.Count > 0)
{
if (IsDebug)
{
string message = string.Format("Received {0} records. ", records.Count);
Console.WriteLine(message);
foreach (IDebugObserver observer in _debugObservers)
{
observer.WriteDebug(message);
}
}
foreach (Amazon.Kinesis.Model.Record record in records)
{
if (!IsRunning)
{
break;
}
Thread.Sleep(1);
string json = Encoding.UTF8.GetString(record.Data.ToArray());
if (IsDebug)
{
string message = "Json string: " + json;
Console.WriteLine(message);
foreach (IDebugObserver observer in _debugObservers)
{
observer.WriteDebug(message);
}
}
T obj = JsonConvert.DeserializeObject <T>(json);
if (obj != null && _dataHandler != null)
{
_dataHandler(obj);
}
}
}
iteratorId = nextIterator;
}
if (!IsRunning)
{
break;
}
}
}
}
private async Task ReadRefPersonStream()
{
Dictionary <string, Models.RefPerson> refPersons = new Dictionary <string, Models.RefPerson>();
refPersons.Clear();
dataGridDetectedPersons.ItemsSource = refPersons.Values;
dataGridDetectedPersons.Items.Refresh();
string streamArn = Models.MyAWSConfigs.DynamodbRefPersonTableStreamArn;
//int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
//&& processedRecordCount < maxItemCount
while (currentShardIter != null)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
string changedRefPersonId = record.Dynamodb.NewImage["id"].S;
string changedRefPersonStatus = record.Dynamodb.NewImage["status"].N.ToString();
string changedRefPersonName = record.Dynamodb.NewImage["name"].S;
string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S;
string changedRefPersonCamera = record.Dynamodb.NewImage["camera"].S;
Models.RefPerson refPerson = new Models.RefPerson();
refPerson.id = changedRefPersonId;
refPerson.name = changedRefPersonName;
refPerson.status = (changedRefPersonStatus == "1") ? true : false;
refPerson.description = changedRefPersonDescription;
refPerson.camera = changedRefPersonCamera;
refPerson.lastLocation = cameras[changedRefPersonCamera].location;
string directoryPath = "Resources/Images/";
if (!File.Exists(directoryPath + refPerson.id))
{
Models.S3Bucket.DownloadFile(refPerson.id, Models.MyAWSConfigs.RefImagesBucketName);
}
string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\";
Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id);
refPerson.image = new BitmapImage(fileUri);
if (refPerson.status)
{
if (!refPersons.ContainsKey(refPerson.id))
{
refPersons.Add(refPerson.id, refPerson);
}
}
else
{
if (refPersons.ContainsKey(refPerson.id))
{
refPersons.Remove(refPerson.id);
}
}
dataGridDetectedPersons.ItemsSource = refPersons.Values;
dataGridDetectedPersons.Items.Refresh();
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
private async Task ReadHistoryStream()
{
string streamArn = Models.MyAWSConfigs.DynamodbHistoryTableStreamArn;
DateTime notifyTime = DateTime.Now;
notifyTime = DateTime.Now.AddMilliseconds(notifyInterval);
//int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
//&& processedRecordCount < maxItemCount
while (currentShardIter != null)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
if (DateTime.Now >= notifyTime)
{
string id = record.Dynamodb.NewImage["id"].S;
string name = record.Dynamodb.NewImage["name"].S;
string camId = record.Dynamodb.NewImage["cameraId"].S;
string camLocation = cameras[camId].location;
string notification = $"{name} has been detected by camera {camId} at {camLocation}";
string trayMessage = $"{name} has been detected";
AddNewNotification(notification, id).Wait();
notifyTime = DateTime.Now.AddMilliseconds(notifyInterval);
notifyIcon.Visible = true;
notifyIcon.ShowBalloonTip(1000, "New Person Detected", trayMessage, System.Windows.Forms.ToolTipIcon.Info);
}
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
