static async Task Main(string[] args)
{
Environment.SetEnvironmentVariable("AWS_PROFILE", "testuser");
var streamsClient = new AmazonDynamoDBStreamsClient();
var dynamoDbClient = new AmazonDynamoDBClient();
var tableDecriptor = await dynamoDbClient.DescribeTableAsync("GameScores");
var streamArn = tableDecriptor.Table.LatestStreamArn;
string exclusiveStartShardId = null;
while (true)
{
var result = await streamsClient.DescribeStreamAsync(new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = exclusiveStartShardId
});
var shards = result.StreamDescription.Shards;
foreach (var shard in shards)
{
var shardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON
};
var shardIteratorResponse = await streamsClient.GetShardIteratorAsync(shardIteratorRequest);
var currentShardIterator = shardIteratorResponse.ShardIterator;
while (currentShardIterator != null)
{
var recordsResponse = await streamsClient.GetRecordsAsync(new GetRecordsRequest()
{
ShardIterator = currentShardIterator
});
foreach (var record in recordsResponse.Records)
{
var data = record.Dynamodb;
Console.WriteLine("Table changed");
}
currentShardIterator = recordsResponse.NextShardIterator;
}
}
}
}
private async Task ReadStream()
{
List <Models.RefPerson> refPersons = new List <Models.RefPerson>();
dataGridDetectedPersons.ItemsSource = refPersons;
dataGridDetectedPersons.Items.Refresh();
string streamArn = "arn:aws:dynamodb:ap-southeast-2:358403828169:table/ref_persons/stream/2019-11-18T05:31:40.045";
int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.dynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
while (currentShardIter != null && processedRecordCount < maxItemCount)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
string changedRefPersonId = record.Dynamodb.NewImage["id"].S;
string changedRefPersonStatus = record.Dynamodb.NewImage["status"].N.ToString();
string changedRefPersonName = record.Dynamodb.NewImage["name"].S;
string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S;
//Console.WriteLine($"{changedRefPersonId}:{changedRefPersonStatus}:{changedRefPersonName}:{changedRefPersonDescription}");
Models.RefPerson refPerson = new Models.RefPerson();
refPerson.id = changedRefPersonId;
refPerson.name = changedRefPersonName;
refPerson.status = (changedRefPersonStatus == "1")?true:false;
refPerson.description = changedRefPersonDescription;
refPerson.camera = "mobile_stream_1";
string directoryPath = "Resources/Images/";
if (!File.Exists(directoryPath + refPerson.id))
{
Models.S3Bucket.DownloadFile(refPerson.id);
}
string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\";
Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id);
refPerson.image = new BitmapImage(fileUri);
if (refPerson.status)
{
if (refPersons.FindAll(p => p.id == refPerson.id).Count == 0)
{
refPersons.Add(refPerson);
}
}
else
{
refPersons.RemoveAll(p => p.id == refPerson.id);
}
dataGridDetectedPersons.ItemsSource = refPersons;
dataGridDetectedPersons.Items.Refresh();
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
public async Task Test1()
{
// Transaction put with an outbox message.
var transactWriteItems = new List <TransactWriteItem>();
for (int i = 0; i < 5; i++)
{
var orgDoc = new Document
{
["Id"] = 123,
["Name"] = $"org-123-{i}",
["Foo"] = Document.FromJson("{\"plot\" : \"Nothing happens at all.\",\"rating\" : 0}")
};
transactWriteItems.Add(new TransactWriteItem
{
Put = new Put
{
TableName = OrgsTableName,
Item = orgDoc.ToAttributeMap()
}
});
var outboxDoc = new Document
{
["Id"] = $"123-{i}",
["Created"] = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
["Body"] = Document.FromJson("{\"plot\" : \"Nothing happens at all.\",\"rating\" : 0}")
};
transactWriteItems.Add(new TransactWriteItem
{
Put = new Put
{
TableName = OutboxTableName,
Item = outboxDoc.ToAttributeMap()
}
});
}
var transactWriteItemsRequest = new TransactWriteItemsRequest
{
TransactItems = transactWriteItems,
};
await _client.TransactWriteItemsAsync(transactWriteItemsRequest);
// Read the changes to outbox change stream
string lastEvaluatedShardId = null;
do
{
var describeStreamRequest = new DescribeStreamRequest
{
StreamArn = _tableLatestStreamArn,
ExclusiveStartShardId = lastEvaluatedShardId
};
var describeStreamResponse = await _streamsClient.DescribeStreamAsync(describeStreamRequest);
foreach (var shard in describeStreamResponse.StreamDescription.Shards)
{
var getShardIteratorRequest = new GetShardIteratorRequest
{
StreamArn = _tableLatestStreamArn,
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON
};
var getShardIteratorResponse = await _streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
var currentShardIterator = getShardIteratorResponse.ShardIterator;
var iterations = 0; // loop will continue for some time until the stream shard is closed, this just short circuits things for the test.
while (currentShardIterator != null && iterations < 10)
{
var getRecordsRequest = new GetRecordsRequest
{
ShardIterator = currentShardIterator,
};
var getRecordsResponse = await _streamsClient.GetRecordsAsync(getRecordsRequest);
foreach (var record in getRecordsResponse.Records)
{
_testOutputHelper.WriteLine($"{record.EventID} {record.EventName} {record.EventSource} {record.Dynamodb.NewImage.StreamViewType}");
}
currentShardIterator = getRecordsResponse.NextShardIterator;
iterations++;
}
}
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
private async Task ReadRefPersonStream()
{
Dictionary <string, Models.RefPerson> refPersons = new Dictionary <string, Models.RefPerson>();
refPersons.Clear();
dataGridDetectedPersons.ItemsSource = refPersons.Values;
dataGridDetectedPersons.Items.Refresh();
string streamArn = Models.MyAWSConfigs.DynamodbRefPersonTableStreamArn;
//int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
//&& processedRecordCount < maxItemCount
while (currentShardIter != null)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
string changedRefPersonId = record.Dynamodb.NewImage["id"].S;
string changedRefPersonStatus = record.Dynamodb.NewImage["status"].N.ToString();
string changedRefPersonName = record.Dynamodb.NewImage["name"].S;
string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S;
string changedRefPersonCamera = record.Dynamodb.NewImage["camera"].S;
Models.RefPerson refPerson = new Models.RefPerson();
refPerson.id = changedRefPersonId;
refPerson.name = changedRefPersonName;
refPerson.status = (changedRefPersonStatus == "1") ? true : false;
refPerson.description = changedRefPersonDescription;
refPerson.camera = changedRefPersonCamera;
refPerson.lastLocation = cameras[changedRefPersonCamera].location;
string directoryPath = "Resources/Images/";
if (!File.Exists(directoryPath + refPerson.id))
{
Models.S3Bucket.DownloadFile(refPerson.id, Models.MyAWSConfigs.RefImagesBucketName);
}
string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\";
Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id);
refPerson.image = new BitmapImage(fileUri);
if (refPerson.status)
{
if (!refPersons.ContainsKey(refPerson.id))
{
refPersons.Add(refPerson.id, refPerson);
}
}
else
{
if (refPersons.ContainsKey(refPerson.id))
{
refPersons.Remove(refPerson.id);
}
}
dataGridDetectedPersons.ItemsSource = refPersons.Values;
dataGridDetectedPersons.Items.Refresh();
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
private async Task ReadHistoryStream()
{
string streamArn = Models.MyAWSConfigs.DynamodbHistoryTableStreamArn;
DateTime notifyTime = DateTime.Now;
notifyTime = DateTime.Now.AddMilliseconds(notifyInterval);
//int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
//&& processedRecordCount < maxItemCount
while (currentShardIter != null)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
if (DateTime.Now >= notifyTime)
{
string id = record.Dynamodb.NewImage["id"].S;
string name = record.Dynamodb.NewImage["name"].S;
string camId = record.Dynamodb.NewImage["cameraId"].S;
string camLocation = cameras[camId].location;
string notification = $"{name} has been detected by camera {camId} at {camLocation}";
string trayMessage = $"{name} has been detected";
AddNewNotification(notification, id).Wait();
notifyTime = DateTime.Now.AddMilliseconds(notifyInterval);
notifyIcon.Visible = true;
notifyIcon.ShowBalloonTip(1000, "New Person Detected", trayMessage, System.Windows.Forms.ToolTipIcon.Info);
}
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
public static async Task TestStreamAsync()
{
var config = new DynamoDBContextConfig
{
Conversion = DynamoDBEntryConversion.V2,
ConsistentRead = true
};
var Context = new DynamoDBContext(new AmazonDynamoDBClient());
#region test_stream_read
// Save list to table
var list = new TODOList
{
User = "******",
ListId = Guid.NewGuid().ToString(),
Complete = false,
Name = "StreamTest",
CreateDate = DateTime.UtcNow,
UpdateDate = DateTime.UtcNow,
Items = new List <TODOListItem>
{
new TODOListItem {
Description = "Task1", Complete = true
},
new TODOListItem {
Description = "Task2", Complete = false
}
}
};
await Context.SaveAsync(list);
using (var streamClient = new AmazonDynamoDBStreamsClient())
{
// Function for reading records continuously from a shard.
Func <string, CancellationToken, Task> shardReader = async(iterator, token) =>
{
while (!token.IsCancellationRequested)
{
var response = (await streamClient.GetRecordsAsync(new GetRecordsRequest {
ShardIterator = iterator
}));
// Update position in shard iterator
iterator = response.NextShardIterator;
// This is what you would write in a Lambda function processing DynamoDB Streams.
foreach (var record in response.Records)
{
var newVersion = record.Dynamodb.NewImage;
Console.WriteLine($"Item read: {newVersion["User"].S}/{newVersion["ListId"].S}");
}
}
};
// Find the arn for the DynamoDB Stream
var streamArn = (await streamClient.ListStreamsAsync(new ListStreamsRequest {
TableName = "TODOList"
}))
.Streams.FirstOrDefault()?.StreamArn;
Console.WriteLine($"The stream arn is {streamArn} shards");
// A stream is made of shards.
var shards = (await streamClient.DescribeStreamAsync(new DescribeStreamRequest {
StreamArn = streamArn
}))
.StreamDescription.Shards;
Console.WriteLine($"The stream currently has {shards.Count} shards");
// Execute a separate reader for each shard.
var cancelSource = new CancellationTokenSource();
var readerTasks = new Task[shards.Count];
for (int i = 0; i < readerTasks.Length; i++)
{
var shardIteratorRequest = new GetShardIteratorRequest
{
StreamArn = streamArn,
ShardId = shards[i].ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON
};
var shardIterator = (await streamClient.GetShardIteratorAsync(shardIteratorRequest))
.ShardIterator;
Console.WriteLine($"Shard Iterator {i}: {shardIterator}");
readerTasks[i] = shardReader(shardIterator, cancelSource.Token);
}
Task.WaitAll(readerTasks, TimeSpan.FromSeconds(5));
cancelSource.Cancel();
}
#endregion
}
