예제 #1
0
        static async Task Main(string[] args)
        {
            Environment.SetEnvironmentVariable("AWS_PROFILE", "testuser");

            var streamsClient = new AmazonDynamoDBStreamsClient();

            var dynamoDbClient = new AmazonDynamoDBClient();

            var tableDecriptor = await dynamoDbClient.DescribeTableAsync("GameScores");

            var streamArn = tableDecriptor.Table.LatestStreamArn;

            string exclusiveStartShardId = null;

            while (true)
            {
                var result = await streamsClient.DescribeStreamAsync(new DescribeStreamRequest()
                {
                    StreamArn             = streamArn,
                    ExclusiveStartShardId = exclusiveStartShardId
                });

                var shards = result.StreamDescription.Shards;

                foreach (var shard in shards)
                {
                    var shardIteratorRequest = new GetShardIteratorRequest()
                    {
                        StreamArn         = streamArn,
                        ShardId           = shard.ShardId,
                        ShardIteratorType = ShardIteratorType.TRIM_HORIZON
                    };

                    var shardIteratorResponse = await streamsClient.GetShardIteratorAsync(shardIteratorRequest);

                    var currentShardIterator = shardIteratorResponse.ShardIterator;

                    while (currentShardIterator != null)
                    {
                        var recordsResponse = await streamsClient.GetRecordsAsync(new GetRecordsRequest()
                        {
                            ShardIterator = currentShardIterator
                        });

                        foreach (var record in recordsResponse.Records)
                        {
                            var data = record.Dynamodb;

                            Console.WriteLine("Table changed");
                        }

                        currentShardIterator = recordsResponse.NextShardIterator;
                    }
                }
            }
        }
예제 #2
0
        private async Task ReadStream()
        {
            List <Models.RefPerson> refPersons = new List <Models.RefPerson>();

            dataGridDetectedPersons.ItemsSource = refPersons;
            dataGridDetectedPersons.Items.Refresh();

            string streamArn    = "arn:aws:dynamodb:ap-southeast-2:358403828169:table/ref_persons/stream/2019-11-18T05:31:40.045";
            int    maxItemCount = 100;

            try
            {
                AmazonDynamoDBStreamsClient streamsClient;
                using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.dynamodbRegion))
                {
                    String lastEvaluatedShardId = null;

                    do
                    {
                        DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
                        {
                            StreamArn             = streamArn,
                            ExclusiveStartShardId = lastEvaluatedShardId,
                        };

                        DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);

                        List <Shard> shards = describeStreamResponse.StreamDescription.Shards;

                        // Process each shard on this page

                        foreach (Shard shard in shards)
                        {
                            String shardId = shard.ShardId;

                            // Get an iterator for the current shard

                            GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
                            {
                                StreamArn         = streamArn,
                                ShardId           = shardId,
                                ShardIteratorType = ShardIteratorType.LATEST,
                            };

                            GetShardIteratorResponse getShardIteratorResponse =
                                await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);

                            String currentShardIter = getShardIteratorResponse.ShardIterator;

                            int processedRecordCount = 0;
                            while (currentShardIter != null && processedRecordCount < maxItemCount)
                            {
                                // Use the shard iterator to read the stream records

                                GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
                                {
                                    ShardIterator = currentShardIter
                                };

                                GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);

                                List <Record> records = getRecordsResponse.Records;

                                foreach (Record record in records)
                                {
                                    foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
                                    {
                                        string changedRefPersonId          = record.Dynamodb.NewImage["id"].S;
                                        string changedRefPersonStatus      = record.Dynamodb.NewImage["status"].N.ToString();
                                        string changedRefPersonName        = record.Dynamodb.NewImage["name"].S;
                                        string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S;

                                        //Console.WriteLine($"{changedRefPersonId}:{changedRefPersonStatus}:{changedRefPersonName}:{changedRefPersonDescription}");

                                        Models.RefPerson refPerson = new Models.RefPerson();

                                        refPerson.id          = changedRefPersonId;
                                        refPerson.name        = changedRefPersonName;
                                        refPerson.status      = (changedRefPersonStatus == "1")?true:false;
                                        refPerson.description = changedRefPersonDescription;
                                        refPerson.camera      = "mobile_stream_1";

                                        string directoryPath = "Resources/Images/";

                                        if (!File.Exists(directoryPath + refPerson.id))
                                        {
                                            Models.S3Bucket.DownloadFile(refPerson.id);
                                        }

                                        string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\";

                                        Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id);

                                        refPerson.image = new BitmapImage(fileUri);

                                        if (refPerson.status)
                                        {
                                            if (refPersons.FindAll(p => p.id == refPerson.id).Count == 0)
                                            {
                                                refPersons.Add(refPerson);
                                            }
                                        }
                                        else
                                        {
                                            refPersons.RemoveAll(p => p.id == refPerson.id);
                                        }

                                        dataGridDetectedPersons.ItemsSource = refPersons;
                                        dataGridDetectedPersons.Items.Refresh();
                                    }
                                }
                                processedRecordCount += records.Count;
                                currentShardIter      = getRecordsResponse.NextShardIterator;
                            }
                        }

                        // If LastEvaluatedShardId is set, then there is
                        // at least one more page of shard IDs to retrieve
                        lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
                    } while (lastEvaluatedShardId != null);
                }
            }
            catch (AmazonDynamoDBException e)
            {
                Console.WriteLine("AmazonDynamoDBException: " + e);
            }
            catch (Exception e)
            {
                Console.WriteLine("Error: " + e);
            }
        }
예제 #3
0
파일: UnitTest1.cs 프로젝트: valcinei/lab
        public async Task Test1()
        {
            // Transaction put with an outbox message.
            var transactWriteItems = new List <TransactWriteItem>();

            for (int i = 0; i < 5; i++)
            {
                var orgDoc = new Document
                {
                    ["Id"]   = 123,
                    ["Name"] = $"org-123-{i}",
                    ["Foo"]  = Document.FromJson("{\"plot\" : \"Nothing happens at all.\",\"rating\" : 0}")
                };

                transactWriteItems.Add(new TransactWriteItem
                {
                    Put = new Put
                    {
                        TableName = OrgsTableName,
                        Item      = orgDoc.ToAttributeMap()
                    }
                });

                var outboxDoc = new Document
                {
                    ["Id"]      = $"123-{i}",
                    ["Created"] = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
                    ["Body"]    = Document.FromJson("{\"plot\" : \"Nothing happens at all.\",\"rating\" : 0}")
                };

                transactWriteItems.Add(new TransactWriteItem
                {
                    Put = new Put
                    {
                        TableName = OutboxTableName,
                        Item      = outboxDoc.ToAttributeMap()
                    }
                });
            }

            var transactWriteItemsRequest = new TransactWriteItemsRequest
            {
                TransactItems = transactWriteItems,
            };
            await _client.TransactWriteItemsAsync(transactWriteItemsRequest);


            // Read the changes to outbox change stream
            string lastEvaluatedShardId = null;

            do
            {
                var describeStreamRequest = new DescribeStreamRequest
                {
                    StreamArn             = _tableLatestStreamArn,
                    ExclusiveStartShardId = lastEvaluatedShardId
                };

                var describeStreamResponse = await _streamsClient.DescribeStreamAsync(describeStreamRequest);

                foreach (var shard in describeStreamResponse.StreamDescription.Shards)
                {
                    var getShardIteratorRequest = new GetShardIteratorRequest
                    {
                        StreamArn         = _tableLatestStreamArn,
                        ShardId           = shard.ShardId,
                        ShardIteratorType = ShardIteratorType.TRIM_HORIZON
                    };
                    var getShardIteratorResponse = await _streamsClient.GetShardIteratorAsync(getShardIteratorRequest);

                    var currentShardIterator = getShardIteratorResponse.ShardIterator;

                    var iterations = 0; // loop will continue for some time until the stream shard is closed, this just short circuits things for the test.
                    while (currentShardIterator != null && iterations < 10)
                    {
                        var getRecordsRequest = new GetRecordsRequest
                        {
                            ShardIterator = currentShardIterator,
                        };
                        var getRecordsResponse = await _streamsClient.GetRecordsAsync(getRecordsRequest);

                        foreach (var record in getRecordsResponse.Records)
                        {
                            _testOutputHelper.WriteLine($"{record.EventID} {record.EventName} {record.EventSource} {record.Dynamodb.NewImage.StreamViewType}");
                        }

                        currentShardIterator = getRecordsResponse.NextShardIterator;
                        iterations++;
                    }
                }

                lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
            } while (lastEvaluatedShardId != null);
        }
예제 #4
0
        private async Task ReadRefPersonStream()
        {
            Dictionary <string, Models.RefPerson> refPersons = new Dictionary <string, Models.RefPerson>();

            refPersons.Clear();
            dataGridDetectedPersons.ItemsSource = refPersons.Values;
            dataGridDetectedPersons.Items.Refresh();

            string streamArn = Models.MyAWSConfigs.DynamodbRefPersonTableStreamArn;

            //int maxItemCount = 100;

            try
            {
                AmazonDynamoDBStreamsClient streamsClient;
                using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion))
                {
                    String lastEvaluatedShardId = null;

                    do
                    {
                        DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
                        {
                            StreamArn             = streamArn,
                            ExclusiveStartShardId = lastEvaluatedShardId,
                        };

                        DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);

                        List <Shard> shards = describeStreamResponse.StreamDescription.Shards;

                        // Process each shard on this page

                        foreach (Shard shard in shards)
                        {
                            String shardId = shard.ShardId;

                            // Get an iterator for the current shard

                            GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
                            {
                                StreamArn         = streamArn,
                                ShardId           = shardId,
                                ShardIteratorType = ShardIteratorType.LATEST,
                            };

                            GetShardIteratorResponse getShardIteratorResponse =
                                await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);

                            String currentShardIter = getShardIteratorResponse.ShardIterator;

                            int processedRecordCount = 0;
                            //&& processedRecordCount < maxItemCount
                            while (currentShardIter != null)
                            {
                                // Use the shard iterator to read the stream records

                                GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
                                {
                                    ShardIterator = currentShardIter
                                };

                                GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);

                                List <Record> records = getRecordsResponse.Records;

                                foreach (Record record in records)
                                {
                                    foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
                                    {
                                        string changedRefPersonId          = record.Dynamodb.NewImage["id"].S;
                                        string changedRefPersonStatus      = record.Dynamodb.NewImage["status"].N.ToString();
                                        string changedRefPersonName        = record.Dynamodb.NewImage["name"].S;
                                        string changedRefPersonDescription = record.Dynamodb.NewImage["description"].S;
                                        string changedRefPersonCamera      = record.Dynamodb.NewImage["camera"].S;

                                        Models.RefPerson refPerson = new Models.RefPerson();

                                        refPerson.id           = changedRefPersonId;
                                        refPerson.name         = changedRefPersonName;
                                        refPerson.status       = (changedRefPersonStatus == "1") ? true : false;
                                        refPerson.description  = changedRefPersonDescription;
                                        refPerson.camera       = changedRefPersonCamera;
                                        refPerson.lastLocation = cameras[changedRefPersonCamera].location;

                                        string directoryPath = "Resources/Images/";

                                        if (!File.Exists(directoryPath + refPerson.id))
                                        {
                                            Models.S3Bucket.DownloadFile(refPerson.id, Models.MyAWSConfigs.RefImagesBucketName);
                                        }

                                        string exeDirectory = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\";

                                        Uri fileUri = new Uri(exeDirectory + directoryPath + refPerson.id);

                                        refPerson.image = new BitmapImage(fileUri);

                                        if (refPerson.status)
                                        {
                                            if (!refPersons.ContainsKey(refPerson.id))
                                            {
                                                refPersons.Add(refPerson.id, refPerson);
                                            }
                                        }
                                        else
                                        {
                                            if (refPersons.ContainsKey(refPerson.id))
                                            {
                                                refPersons.Remove(refPerson.id);
                                            }
                                        }

                                        dataGridDetectedPersons.ItemsSource = refPersons.Values;
                                        dataGridDetectedPersons.Items.Refresh();
                                    }
                                }
                                processedRecordCount += records.Count;
                                currentShardIter      = getRecordsResponse.NextShardIterator;
                            }
                        }

                        // If LastEvaluatedShardId is set, then there is
                        // at least one more page of shard IDs to retrieve
                        lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
                    } while (lastEvaluatedShardId != null);
                }
            }
            catch (AmazonDynamoDBException e)
            {
                Console.WriteLine("AmazonDynamoDBException: " + e);
            }
            catch (Exception e)
            {
                Console.WriteLine("Error: " + e);
            }
        }
예제 #5
0
        private async Task ReadHistoryStream()
        {
            string streamArn = Models.MyAWSConfigs.DynamodbHistoryTableStreamArn;

            DateTime notifyTime = DateTime.Now;

            notifyTime = DateTime.Now.AddMilliseconds(notifyInterval);

            //int maxItemCount = 100;

            try
            {
                AmazonDynamoDBStreamsClient streamsClient;
                using (streamsClient = new AmazonDynamoDBStreamsClient(Models.MyAWSConfigs.DynamodbRegion))
                {
                    String lastEvaluatedShardId = null;

                    do
                    {
                        DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
                        {
                            StreamArn             = streamArn,
                            ExclusiveStartShardId = lastEvaluatedShardId,
                        };

                        DescribeStreamResponse describeStreamResponse = await streamsClient.DescribeStreamAsync(describeStreamRequest);

                        List <Shard> shards = describeStreamResponse.StreamDescription.Shards;

                        // Process each shard on this page

                        foreach (Shard shard in shards)
                        {
                            String shardId = shard.ShardId;

                            // Get an iterator for the current shard

                            GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
                            {
                                StreamArn         = streamArn,
                                ShardId           = shardId,
                                ShardIteratorType = ShardIteratorType.LATEST,
                            };

                            GetShardIteratorResponse getShardIteratorResponse =
                                await streamsClient.GetShardIteratorAsync(getShardIteratorRequest);

                            String currentShardIter = getShardIteratorResponse.ShardIterator;

                            int processedRecordCount = 0;
                            //&& processedRecordCount < maxItemCount
                            while (currentShardIter != null)
                            {
                                // Use the shard iterator to read the stream records

                                GetRecordsRequest getRecordsRequest = new GetRecordsRequest()
                                {
                                    ShardIterator = currentShardIter
                                };

                                GetRecordsResponse getRecordsResponse = await streamsClient.GetRecordsAsync(getRecordsRequest);

                                List <Record> records = getRecordsResponse.Records;

                                foreach (Record record in records)
                                {
                                    foreach (KeyValuePair <string, AttributeValue> newImage in record.Dynamodb.NewImage)
                                    {
                                        if (DateTime.Now >= notifyTime)
                                        {
                                            string id           = record.Dynamodb.NewImage["id"].S;
                                            string name         = record.Dynamodb.NewImage["name"].S;
                                            string camId        = record.Dynamodb.NewImage["cameraId"].S;
                                            string camLocation  = cameras[camId].location;
                                            string notification = $"{name} has been detected by camera {camId} at {camLocation}";
                                            string trayMessage  = $"{name} has been detected";

                                            AddNewNotification(notification, id).Wait();

                                            notifyTime = DateTime.Now.AddMilliseconds(notifyInterval);

                                            notifyIcon.Visible = true;
                                            notifyIcon.ShowBalloonTip(1000, "New Person Detected", trayMessage, System.Windows.Forms.ToolTipIcon.Info);
                                        }
                                    }
                                }
                                processedRecordCount += records.Count;
                                currentShardIter      = getRecordsResponse.NextShardIterator;
                            }
                        }

                        // If LastEvaluatedShardId is set, then there is
                        // at least one more page of shard IDs to retrieve
                        lastEvaluatedShardId = describeStreamResponse.StreamDescription.LastEvaluatedShardId;
                    } while (lastEvaluatedShardId != null);
                }
            }
            catch (AmazonDynamoDBException e)
            {
                Console.WriteLine("AmazonDynamoDBException: " + e);
            }
            catch (Exception e)
            {
                Console.WriteLine("Error: " + e);
            }
        }
예제 #6
0
        public static async Task TestStreamAsync()
        {
            var config = new DynamoDBContextConfig
            {
                Conversion     = DynamoDBEntryConversion.V2,
                ConsistentRead = true
            };
            var Context = new DynamoDBContext(new AmazonDynamoDBClient());

            #region test_stream_read

            // Save list to table
            var list = new TODOList
            {
                User       = "******",
                ListId     = Guid.NewGuid().ToString(),
                Complete   = false,
                Name       = "StreamTest",
                CreateDate = DateTime.UtcNow,
                UpdateDate = DateTime.UtcNow,
                Items      = new List <TODOListItem>
                {
                    new TODOListItem {
                        Description = "Task1", Complete = true
                    },
                    new TODOListItem {
                        Description = "Task2", Complete = false
                    }
                }
            };

            await Context.SaveAsync(list);

            using (var streamClient = new AmazonDynamoDBStreamsClient())
            {
                // Function for reading records continuously from a shard.
                Func <string, CancellationToken, Task> shardReader = async(iterator, token) =>
                {
                    while (!token.IsCancellationRequested)
                    {
                        var response = (await streamClient.GetRecordsAsync(new GetRecordsRequest {
                            ShardIterator = iterator
                        }));

                        // Update position in shard iterator
                        iterator = response.NextShardIterator;

                        // This is what you would write in a Lambda function processing DynamoDB Streams.
                        foreach (var record in response.Records)
                        {
                            var newVersion = record.Dynamodb.NewImage;
                            Console.WriteLine($"Item read: {newVersion["User"].S}/{newVersion["ListId"].S}");
                        }
                    }
                };


                // Find the arn for the DynamoDB Stream
                var streamArn = (await streamClient.ListStreamsAsync(new ListStreamsRequest {
                    TableName = "TODOList"
                }))
                                .Streams.FirstOrDefault()?.StreamArn;

                Console.WriteLine($"The stream arn is {streamArn} shards");

                // A stream is made of shards.
                var shards = (await streamClient.DescribeStreamAsync(new DescribeStreamRequest {
                    StreamArn = streamArn
                }))
                             .StreamDescription.Shards;

                Console.WriteLine($"The stream currently has {shards.Count} shards");


                // Execute a separate reader for each shard.
                var cancelSource = new CancellationTokenSource();
                var readerTasks  = new Task[shards.Count];
                for (int i = 0; i < readerTasks.Length; i++)
                {
                    var shardIteratorRequest = new GetShardIteratorRequest
                    {
                        StreamArn         = streamArn,
                        ShardId           = shards[i].ShardId,
                        ShardIteratorType = ShardIteratorType.TRIM_HORIZON
                    };

                    var shardIterator = (await streamClient.GetShardIteratorAsync(shardIteratorRequest))
                                        .ShardIterator;
                    Console.WriteLine($"Shard Iterator {i}: {shardIterator}");

                    readerTasks[i] = shardReader(shardIterator, cancelSource.Token);
                }

                Task.WaitAll(readerTasks, TimeSpan.FromSeconds(5));
                cancelSource.Cancel();
            }
            #endregion
        }