public UnitTest1(ITestOutputHelper testOutputHelper)
{
_testOutputHelper = testOutputHelper;
var containerService = new Builder()
.UseContainer()
.WithName("dynamodb-outbox")
.UseImage("amazon/dynamodb-local:1.13.0")
.KeepRunning()
.ReuseIfExists()
.Command("", "-jar", "DynamoDBLocal.jar", "-inMemory", "-sharedDb")
.ExposePort(8000, 8000)
.WaitForPort("8000/tcp", 5000, "127.0.0.1")
.Build();
_service = containerService.Start();
var awsCredentials = new BasicAWSCredentials("ignored", "ignored");
var dynamoDBConfig = new AmazonDynamoDBConfig
{
ServiceURL = "http://localhost:8000",
};
_client = new AmazonDynamoDBClient(awsCredentials, dynamoDBConfig);
var streamsConfig = new AmazonDynamoDBStreamsConfig
{
ServiceURL = dynamoDBConfig.ServiceURL
};
_streamsClient = new AmazonDynamoDBStreamsClient(awsCredentials, streamsConfig);
}
static async Task Main(string[] args)
{
Environment.SetEnvironmentVariable("AWS_PROFILE", "testuser");
var streamsClient = new AmazonDynamoDBStreamsClient();
var dynamoDbClient = new AmazonDynamoDBClient();
var tableDecriptor = await dynamoDbClient.DescribeTableAsync("GameScores");
var streamArn = tableDecriptor.Table.LatestStreamArn;
string exclusiveStartShardId = null;
while (true)
{
var result = await streamsClient.DescribeStreamAsync(new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = exclusiveStartShardId
});
var shards = result.StreamDescription.Shards;
foreach (var shard in shards)
{
var shardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON
};
var shardIteratorResponse = await streamsClient.GetShardIteratorAsync(shardIteratorRequest);
var currentShardIterator = shardIteratorResponse.ShardIterator;
while (currentShardIterator != null)
{
var recordsResponse = await streamsClient.GetRecordsAsync(new GetRecordsRequest()
{
ShardIterator = currentShardIterator
});
foreach (var record in recordsResponse.Records)
{
var data = record.Dynamodb;
Console.WriteLine("Table changed");
}
currentShardIterator = recordsResponse.NextShardIterator;
}
}
}
}
public void TestDynamoDBStreamWithServiceURL()
{
var config = new AmazonDynamoDBStreamsConfig();
config.ServiceURL = "https://streams.dynamodb.us-east-1.amazonaws.com/";
using (var client = new AmazonDynamoDBStreamsClient(config))
{
var response = client.ListStreams();
Assert.IsNotNull(response.Streams);
}
}
protected IAmazonDynamoDBStreams CreateClient(AWSCredentials credentials, RegionEndpoint region)
{
var config = new AmazonDynamoDBStreamsConfig {
RegionEndpoint = region
};
Amazon.PowerShell.Utils.Common.PopulateConfig(this, config);
this.CustomizeClientConfig(config);
var client = new AmazonDynamoDBStreamsClient(credentials, config);
client.BeforeRequestEvent += RequestEventHandler;
client.AfterResponseEvent += ResponseEventHandler;
return(client);
}
public AWSDynamoDBStream(BasicAWSCredentials basicAWSCredentials, RegionEndpoint regionEndpoint, string tableName, AWSDynamoDBIteratorType type)
{
this.TableName = tableName;
this.AmazonDynamoDBStreamsClient = new AmazonDynamoDBStreamsClient(basicAWSCredentials, regionEndpoint);
this.AWSDynamoDBIteratorType = type;
var listStreams = AmazonDynamoDBStreamsClient.ListStreams(new ListStreamsRequest()
{
TableName = this.TableName
});
this.StreamArn = listStreams.Streams.First().StreamArn;
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = this.StreamArn
};
var describeStreamResponse = this.AmazonDynamoDBStreamsClient.DescribeStream(describeStreamRequest);
var shards = describeStreamResponse.StreamDescription.Shards;
GetShardIteratorRequest getShardIteratorRequest = null;
if (this.AWSDynamoDBIteratorType == AWSDynamoDBIteratorType.TRIM_HORIZON)
{
getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = this.StreamArn,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON,
ShardId = shards.First().ShardId,
};
}
if (this.AWSDynamoDBIteratorType == AWSDynamoDBIteratorType.LATEST)
{
getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = this.StreamArn,
ShardIteratorType = ShardIteratorType.LATEST,
ShardId = shards.Last().ShardId,
//SequenceNumber = shards.First().SequenceNumberRange.StartingSequenceNumber
};
}
var shardIteratorResponse = this.AmazonDynamoDBStreamsClient.GetShardIterator(getShardIteratorRequest);
this.LatestShardID = shardIteratorResponse.ShardIterator;
}
public static async Task AddEventSourceAsync()
{
#region add_dynamodb_event_source
using (var streamClient = new AmazonDynamoDBStreamsClient())
using (var lambdaClient = new AmazonLambdaClient())
{
// TODO: Enter Lambda function name, this is most likely: ServerlessTODOListStreamProcessor
var functionName = "";
if (string.IsNullOrEmpty(functionName))
{
Console.Error.WriteLine("You must set the name of the Lambda function you deployed to the \"functionName\" variable");
return;
}
var listRequest = new ListStreamsRequest
{
TableName = "TODOList"
};
var listResponse = await streamClient.ListStreamsAsync(listRequest);
if (listResponse.Streams.Count == 0)
{
Console.Error.WriteLine($"A stream is not enabled for the table {listRequest.TableName}");
return;
}
var streamArn = listResponse.Streams[0].StreamArn;
Console.WriteLine($"Stream ARN is {streamArn}");
var request = new CreateEventSourceMappingRequest
{
FunctionName = functionName,
EventSourceArn = streamArn,
StartingPosition = EventSourcePosition.LATEST,
BatchSize = 100
};
await lambdaClient.CreateEventSourceMappingAsync(request);
Console.WriteLine($"Event source mapping made between stream {streamArn} and function {functionName}");
}
#endregion
}
private async Task ReadStream()
{
List <Models.RefPerson> refPersons = new List <Models.RefPerson>();
dataGridDetectedPersons.ItemsSource = refPersons;
dataGridDetectedPersons.Items.Refresh();
string streamArn = "arn:aws:dynamodb:ap-southeast-2:358403828169:table/ref_persons/stream/2019-11-18T05:31:40.045";
int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.dynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
while (currentShardIter != null && processedRecordCount < maxItemCount)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
string changedRefPersonId = record.Dynamodb.NewImage["id"].S;
string changedRefPersonStatus = record.Dynamodb.NewImage["status"].N.ToString();
string changedRefPersonName = record.Dynamodb.NewImage["name"].S;
string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S;
//Console.WriteLine($"{changedRefPersonId}:{changedRefPersonStatus}:{changedRefPersonName}:{changedRefPersonDescription}");
Models.RefPerson refPerson = new Models.RefPerson();
refPerson.id = changedRefPersonId;
refPerson.name = changedRefPersonName;
refPerson.status = (changedRefPersonStatus == "1")?true:false;
refPerson.description = changedRefPersonDescription;
refPerson.camera = "mobile_stream_1";
string directoryPath = "Resources/Images/";
if (!File.Exists(directoryPath + refPerson.id))
{
Models.S3Bucket.DownloadFile(refPerson.id);
}
string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\";
Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id);
refPerson.image = new BitmapImage(fileUri);
if (refPerson.status)
{
if (refPersons.FindAll(p => p.id == refPerson.id).Count == 0)
{
refPersons.Add(refPerson);
}
}
else
{
refPersons.RemoveAll(p => p.id == refPerson.id);
}
dataGridDetectedPersons.ItemsSource = refPersons;
dataGridDetectedPersons.Items.Refresh();
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
private async Task ReadRefPersonStream()
{
Dictionary <string, Models.RefPerson> refPersons = new Dictionary <string, Models.RefPerson>();
refPersons.Clear();
dataGridDetectedPersons.ItemsSource = refPersons.Values;
dataGridDetectedPersons.Items.Refresh();
string streamArn = Models.MyAWSConfigs.DynamodbRefPersonTableStreamArn;
//int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
//&& processedRecordCount < maxItemCount
while (currentShardIter != null)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
string changedRefPersonId = record.Dynamodb.NewImage["id"].S;
string changedRefPersonStatus = record.Dynamodb.NewImage["status"].N.ToString();
string changedRefPersonName = record.Dynamodb.NewImage["name"].S;
string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S;
string changedRefPersonCamera = record.Dynamodb.NewImage["camera"].S;
Models.RefPerson refPerson = new Models.RefPerson();
refPerson.id = changedRefPersonId;
refPerson.name = changedRefPersonName;
refPerson.status = (changedRefPersonStatus == "1") ? true : false;
refPerson.description = changedRefPersonDescription;
refPerson.camera = changedRefPersonCamera;
refPerson.lastLocation = cameras[changedRefPersonCamera].location;
string directoryPath = "Resources/Images/";
if (!File.Exists(directoryPath + refPerson.id))
{
Models.S3Bucket.DownloadFile(refPerson.id, Models.MyAWSConfigs.RefImagesBucketName);
}
string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\";
Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id);
refPerson.image = new BitmapImage(fileUri);
if (refPerson.status)
{
if (!refPersons.ContainsKey(refPerson.id))
{
refPersons.Add(refPerson.id, refPerson);
}
}
else
{
if (refPersons.ContainsKey(refPerson.id))
{
refPersons.Remove(refPerson.id);
}
}
dataGridDetectedPersons.ItemsSource = refPersons.Values;
dataGridDetectedPersons.Items.Refresh();
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
private async Task ReadHistoryStream()
{
string streamArn = Models.MyAWSConfigs.DynamodbHistoryTableStreamArn;
DateTime notifyTime = DateTime.Now;
notifyTime = DateTime.Now.AddMilliseconds(notifyInterval);
//int maxItemCount = 100;
try
{
AmazonDynamoDBStreamsClient streamsClient;
using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion))
{
String lastEvaluatedShardId = null;
do
{
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
{
StreamArn = streamArn,
ExclusiveStartShardId = lastEvaluatedShardId,
};
DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);
List <Shard> shards = describeStreamResponse.StreamDescription.Shards;
// Process each shard on this page
foreach (Shard shard in shards)
{
String shardId = shard.ShardId;
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
{
StreamArn = streamArn,
ShardId = shardId,
ShardIteratorType = ShardIteratorType.LATEST,
};
GetShardIteratorResponse getShardIteratorResponse =
await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);
String currentShardIter = getShardIteratorResponse.ShardIterator;
int processedRecordCount = 0;
//&& processedRecordCount < maxItemCount
while (currentShardIter != null)
{
// Use the shard iterator to read the stream records
GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
{
ShardIterator = currentShardIter
};
GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);
List <Record> records = getRecordsResponse.Records;
foreach (Record record in records)
{
foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
{
if (DateTime.Now >= notifyTime)
{
string id = record.Dynamodb.NewImage["id"].S;
string name = record.Dynamodb.NewImage["name"].S;
string camId = record.Dynamodb.NewImage["cameraId"].S;
string camLocation = cameras[camId].location;
string notification = $"{name} has been detected by camera {camId} at {camLocation}";
string trayMessage = $"{name} has been detected";
AddNewNotification(notification, id).Wait();
notifyTime = DateTime.Now.AddMilliseconds(notifyInterval);
notifyIcon.Visible = true;
notifyIcon.ShowBalloonTip(1000, "New Person Detected", trayMessage, System.Windows.Forms.ToolTipIcon.Info);
}
}
}
processedRecordCount += records.Count;
currentShardIter = getRecordsResponse.NextShardIterator;
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
} while (lastEvaluatedShardId != null);
}
}
catch (AmazonDynamoDBException e)
{
Console.WriteLine("AmazonDynamoDBException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
public static async Task TestStreamAsync()
{
var config = new DynamoDBContextConfig
{
Conversion = DynamoDBEntryConversion.V2,
ConsistentRead = true
};
var Context = new DynamoDBContext(new AmazonDynamoDBClient());
#region test_stream_read
// Save list to table
var list = new TODOList
{
User = "******",
ListId = Guid.NewGuid().ToString(),
Complete = false,
Name = "StreamTest",
CreateDate = DateTime.UtcNow,
UpdateDate = DateTime.UtcNow,
Items = new List <TODOListItem>
{
new TODOListItem {
Description = "Task1", Complete = true
},
new TODOListItem {
Description = "Task2", Complete = false
}
}
};
await Context.SaveAsync(list);
using (var streamClient = new AmazonDynamoDBStreamsClient())
{
// Function for reading records continuously from a shard.
Func <string, CancellationToken, Task> shardReader = async(iterator, token) =>
{
while (!token.IsCancellationRequested)
{
var response = (await streamClient.GetRecordsAsync(new GetRecordsRequest {
ShardIterator = iterator
}));
// Update position in shard iterator
iterator = response.NextShardIterator;
// This is what you would write in a Lambda function processing DynamoDB Streams.
foreach (var record in response.Records)
{
var newVersion = record.Dynamodb.NewImage;
Console.WriteLine($"Item read: {newVersion["User"].S}/{newVersion["ListId"].S}");
}
}
};
// Find the arn for the DynamoDB Stream
var streamArn = (await streamClient.ListStreamsAsync(new ListStreamsRequest {
TableName = "TODOList"
}))
.Streams.FirstOrDefault()?.StreamArn;
Console.WriteLine($"The stream arn is {streamArn} shards");
// A stream is made of shards.
var shards = (await streamClient.DescribeStreamAsync(new DescribeStreamRequest {
StreamArn = streamArn
}))
.StreamDescription.Shards;
Console.WriteLine($"The stream currently has {shards.Count} shards");
// Execute a separate reader for each shard.
var cancelSource = new CancellationTokenSource();
var readerTasks = new Task[shards.Count];
for (int i = 0; i < readerTasks.Length; i++)
{
var shardIteratorRequest = new GetShardIteratorRequest
{
StreamArn = streamArn,
ShardId = shards[i].ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON
};
var shardIterator = (await streamClient.GetShardIteratorAsync(shardIteratorRequest))
.ShardIterator;
Console.WriteLine($"Shard Iterator {i}: {shardIterator}");
readerTasks[i] = shardReader(shardIterator, cancelSource.Token);
}
Task.WaitAll(readerTasks, TimeSpan.FromSeconds(5));
cancelSource.Cancel();
}
#endregion
}
