示例#1
0
        private T ConvertGetItemResponse <T>(GetItemRequest request, DynamoMetadataType table)
        {
            var response = Exec(() => DynamoDb.GetItem(request), rethrowExceptions: throwNotFoundExceptions);

            if (!response.IsItemSet)
            {
                return(default);
示例#2
0
        public long DescribeItemCount <T>()
        {
            var table    = DynamoMetadata.GetTable <T>();
            var response = Exec(() => DynamoDb.DescribeTable(new DescribeTableRequest(table.Name)));

            return(response.Table.ItemCount);
        }
示例#3
0
        private async Task <List <Table> > ExpandTablePatterns(DynamoDb dynamoDb, string alertingGroupName)
        {
            var tablesWithoutPatterns = dynamoDb.Tables
                                        .Where(t => string.IsNullOrWhiteSpace(t.Pattern))
                                        .ToList();

            var patterns = dynamoDb.Tables
                           .Where(t => !string.IsNullOrWhiteSpace(t.Pattern))
                           .ToList();

            var tablesFromPatterns = new List <Table>();

            foreach (var tablePattern in patterns)
            {
                var matches = await GetPatternMatches(tablePattern, alertingGroupName);

                // filter out duplicates
                matches = matches
                          .Where(match => tablesFromPatterns.All(t => t.Name != match.Name))
                          .ToList();

                matches = matches
                          .Where(match => tablesWithoutPatterns.All(t => t.Name != match.Name))
                          .ToList();

                tablesFromPatterns.AddRange(matches);
            }

            return(tablesWithoutPatterns
                   .Union(tablesFromPatterns)
                   .ToList());
        }
示例#4
0
        public async Task UpdateItemAsync <T>(DynamoUpdateItem update, CancellationToken token = default)
        {
            var request = ToUpdateItemRequest <T>(update);

            await ExecAsync(async() =>
                            await DynamoDb.UpdateItemAsync(request, token).ConfigAwait()).ConfigAwait();
        }
示例#5
0
        public void PutItems <T>(IEnumerable <T> items)
        {
            var table     = DynamoMetadata.GetTable <T>();
            var remaining = items.ToList();

            PopulateMissingHashes(table, remaining);

            while (remaining.Count > 0)
            {
                var batchSize = Math.Min(remaining.Count, MaxWriteBatchSize);
                var nextBatch = remaining.GetRange(0, batchSize);
                remaining.RemoveRange(0, batchSize);

                var putItems = nextBatch.Map(x => new WriteRequest(
                                                 new PutRequest(Converters.ToAttributeValues(this, x, table))));

                var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> > {
                    { table.Name, putItems }
                });

                var response = Exec(() => DynamoDb.BatchWriteItem(request));

                var i = 0;
                while (response.UnprocessedItems.Count > 0)
                {
                    response = Exec(() => DynamoDb.BatchWriteItem(new BatchWriteItemRequest(response.UnprocessedItems)));

                    if (response.UnprocessedItems.Count > 0)
                    {
                        i.SleepBackOffMultiplier();
                    }
                }
            }
        }
        private List <T> ConvertBatchGetItemResponse <T>(DynamoMetadataType table, KeysAndAttributes getItems)
        {
            var to = new List <T>();

            var request = new BatchGetItemRequest(new Dictionary <string, KeysAndAttributes> {
                { table.Name, getItems }
            });

            var response = Exec(() => DynamoDb.BatchGetItem(request));

            List <Dictionary <string, AttributeValue> > results;

            if (response.Responses.TryGetValue(table.Name, out results))
            {
                results.Each(x => to.Add(Converters.FromAttributeValues <T>(table, x)));
            }

            var i = 0;

            while (response.UnprocessedKeys.Count > 0)
            {
                response = Exec(() => DynamoDb.BatchGetItem(new BatchGetItemRequest(response.UnprocessedKeys)));
                if (response.Responses.TryGetValue(table.Name, out results))
                {
                    results.Each(x => to.Add(Converters.FromAttributeValues <T>(table, x)));
                }

                if (response.UnprocessedKeys.Count > 0)
                {
                    i.SleepBackOffMultiplier();
                }
            }

            return(to);
        }
示例#7
0
        public long Increment <T>(object hash, string fieldName, long amount = 1)
        {
            var type    = DynamoMetadata.GetType <T>();
            var request = new UpdateItemRequest
            {
                TableName        = type.Name,
                Key              = Converters.ToAttributeKeyValue(this, type.HashKey, hash),
                AttributeUpdates = new Dictionary <string, AttributeValueUpdate> {
                    {
                        fieldName,
                        new AttributeValueUpdate {
                            Action = AttributeAction.ADD,
                            Value  = new AttributeValue {
                                N = amount.ToString()
                            }
                        }
                    }
                },
                ReturnValues = ReturnValue.ALL_NEW,
            };

            var response = DynamoDb.UpdateItem(request);

            return(response.Attributes.Count > 0
                ? Convert.ToInt64(response.Attributes[fieldName].N)
                : 0);
        }
示例#8
0
        public List <T> Scan <T>(ScanRequest request, int limit)
        {
            var to = new List <T>();

            if (request.Limit == default(int))
            {
                request.Limit = limit;
            }

            ScanResponse response = null;

            do
            {
                if (response != null)
                {
                    request.ExclusiveStartKey = response.LastEvaluatedKey;
                }

                response = Exec(() => DynamoDb.Scan(request));
                var results = response.ConvertAll <T>();

                foreach (var result in results)
                {
                    to.Add(result);

                    if (to.Count >= limit)
                    {
                        break;
                    }
                }
            } while (!response.LastEvaluatedKey.IsEmpty() && to.Count < limit);

            return(to);
        }
示例#9
0
        private async Task <bool> TableExist()
        {
            var tables = await DynamoDb.ListTablesAsync();

            var existTable = tables.TableNames.Contains(TableName);

            return(existTable);
        }
示例#10
0
        public async Task <bool> TableExist(string tableName)
        {
            var tables = await DynamoDb.ListTablesAsync();

            var existTable = tables.TableNames.Contains(tableName);

            return(existTable);
        }
示例#11
0
 protected virtual void Dispose(bool disposing)
 {
     if (DynamoDb != null)
     {
         DynamoDb.Dispose();
         DynamoDb = null;
     }
 }
示例#12
0
 public void ClearEntities()
 {
     //TODO :DI
     using (IDal database = new DynamoDb())
     {
         database.ClearEntities();
     }
 }
示例#13
0
        public void UpdateItem <T>(DynamoUpdateItem update)
        {
            var table   = DynamoMetadata.GetTable <T>();
            var request = new UpdateItemRequest
            {
                TableName        = table.Name,
                Key              = Converters.ToAttributeKeyValue(this, table, update.Hash, update.Range),
                AttributeUpdates = new Dictionary <string, AttributeValueUpdate>(),
                ReturnValues     = ReturnValue.NONE,
            };

            if (update.Put != null)
            {
                foreach (var entry in update.Put)
                {
                    var field = table.GetField(entry.Key);
                    if (field == null)
                    {
                        continue;
                    }

                    request.AttributeUpdates[field.Name] = new AttributeValueUpdate(
                        Converters.ToAttributeValue(this, field.Type, field.DbType, entry.Value), DynamoAttributeAction.Put);
                }
            }

            if (update.Add != null)
            {
                foreach (var entry in update.Add)
                {
                    var field = table.GetField(entry.Key);
                    if (field == null)
                    {
                        continue;
                    }

                    request.AttributeUpdates[field.Name] = new AttributeValueUpdate(
                        Converters.ToAttributeValue(this, field.Type, field.DbType, entry.Value), DynamoAttributeAction.Add);
                }
            }

            if (update.Delete != null)
            {
                foreach (var key in update.Delete)
                {
                    var field = table.GetField(key);
                    if (field == null)
                    {
                        continue;
                    }

                    request.AttributeUpdates[field.Name] = new AttributeValueUpdate(null, DynamoAttributeAction.Delete);
                }
            }

            Exec(() => DynamoDb.UpdateItem(request));
        }
示例#14
0
        private async Task CreateIfNotExist(CreateTableRequest request)
        {
            if (await TableExist())
            {
                return;
            }

            await DynamoDb.CreateTableAsync(request);
        }
示例#15
0
        protected Task <CreateTableResponse> CreateTestTable(string tableName = null)
        {
            var postTableCreateRequest = new CreateTableRequest
            {
                AttributeDefinitions =
                    new List <AttributeDefinition>
                {
                    new AttributeDefinition {
                        AttributeName = nameof(MovieEntity.DirectorId), AttributeType = ScalarAttributeType.S
                    },
                    new AttributeDefinition {
                        AttributeName = nameof(MovieEntity.CreateDate), AttributeType = ScalarAttributeType.S
                    },
                    new AttributeDefinition()
                    {
                        AttributeName = nameof(MovieEntity.MovieId), AttributeType = ScalarAttributeType.S
                    }
                },
                TableName = tableName ?? TestTableName,
                KeySchema =
                    new List <KeySchemaElement>()
                {
                    new KeySchemaElement()
                    {
                        AttributeName = nameof(MovieEntity.DirectorId), KeyType = KeyType.HASH
                    },
                    new KeySchemaElement()
                    {
                        AttributeName = nameof(MovieEntity.CreateDate), KeyType = KeyType.RANGE
                    }
                },
                GlobalSecondaryIndexes = new List <GlobalSecondaryIndex>
                {
                    new GlobalSecondaryIndex
                    {
                        Projection = new Projection {
                            ProjectionType = ProjectionType.ALL
                        },
                        IndexName = TestConstants.MoiveTableMovieIdGsi,
                        KeySchema = new List <KeySchemaElement> {
                            new KeySchemaElement {
                                AttributeName = nameof(MovieEntity.MovieId), KeyType = KeyType.HASH
                            }
                        },
                        ProvisionedThroughput = new ProvisionedThroughput {
                            ReadCapacityUnits = 5, WriteCapacityUnits = 5
                        }
                    }
                },
                ProvisionedThroughput = new ProvisionedThroughput {
                    ReadCapacityUnits = 5, WriteCapacityUnits = 6
                },
            };

            return(DynamoDb.CreateTableAsync(postTableCreateRequest));
        }
示例#16
0
        public void Close()
        {
            if (DynamoDb == null)
            {
                return;
            }

            DynamoDb.Dispose();
            DynamoDb = null;
        }
 public T Get(long Id)
 {
     try
     {
         T item = DynamoDb.GetItem <T>(Id);
         return(item);
     }
     catch  {
         return(null);
     }
 }
        public List <T> GetAll()
        {
            try
            {
                List <T> Items = DynamoDb.GetAll <T>();

                return(Items);
            }
            catch (Exception ex) {
                return(new List <T>());
            }
        }
示例#19
0
        public bool SaveToDatabase(IList <Entity> entities)
        {
            bool result;

            //TODO :DI
            using (IDal database = new DynamoDb())
            {
                result = database.InsertEntities(entities);
            }

            return(result);
        }
        private T ConvertGetItemResponse <T>(GetItemRequest request, DynamoMetadataType table)
        {
            var response = Exec(() => DynamoDb.GetItem(request), rethrowExceptions: throwNotFoundExceptions);

            if (!response.IsItemSet)
            {
                return(default(T));
            }
            var attributeValues = response.Item;

            return(Converters.FromAttributeValues <T>(table, attributeValues));
        }
示例#21
0
 public bool UpdateItem <T>(UpdateExpression <T> update)
 {
     try
     {
         Exec(() => DynamoDb.UpdateItem(update));
         return(true);
     }
     catch (ConditionalCheckFailedException)
     {
         return(false);
     }
 }
示例#22
0
        public Entity SearchEntity(Entity entity)
        {
            //TODO :DI
            Entity result;

            Enum.TryParse(entity.Type, out EntityType type);
            using (IDal database = new DynamoDb())
            {
                result = database.QueryEntity(type, entity.Id, entity.TimeStamp);
            }

            return(result);
        }
示例#23
0
        public async Task <bool> UpdateItemAsync <T>(UpdateExpression <T> update, CancellationToken token = default)
        {
            try
            {
                await ExecAsync(async() =>
                                await DynamoDb.UpdateItemAsync(update, token).ConfigAwait()).ConfigAwait();

                return(true);
            }
            catch (ConditionalCheckFailedException)
            {
                return(false);
            }
        }
        private static WatchmanConfiguration Config()
        {
            var group = new DynamoDb
            {
                Tables = new List <Table>
                {
                    new Table {
                        Pattern = ".*", Threshold = 0.75, MonitorWrites = false
                    }
                }
            };

            return(AlertingGroupData.WrapDynamo(group));
        }
        private static WatchmanConfiguration Config()
        {
            var ag = new DynamoDb
            {
                Tables = new List <Table>
                {
                    new Table {
                        Name = "authentication-tokens"
                    }
                }
            };

            return(AlertingGroupData.WrapDynamo(ag));
        }
        private static WatchmanConfiguration CatchAllConfig()
        {
            var catchAll = new DynamoDb
            {
                Tables = new List <Table>
                {
                    new Table {
                        Pattern = ".*"
                    }
                }
            };

            return(AlertingGroupData.WrapDynamo(catchAll));
        }
示例#27
0
        private static WatchmanConfiguration Config()
        {
            var dynamoConfig = new DynamoDb
            {
                Tables = new List <Table>
                {
                    new Table {
                        Pattern = "nomatchwillbefoundforthis"
                    }
                }
            };

            return(AlertingGroupData.WrapDynamo(dynamoConfig));
        }
        public bool WaitForTablesToBeReady(IEnumerable <string> tableNames, TimeSpan?timeout = null)
        {
            var pendingTables = new List <string>(tableNames);

            if (pendingTables.Count == 0)
            {
                return(true);
            }

            var startAt = DateTime.UtcNow;

            do
            {
                try
                {
                    var responses = pendingTables.Map(x =>
                                                      Exec(() => DynamoDb.DescribeTable(new DescribeTableRequest(x))));

                    foreach (var response in responses)
                    {
                        if (response.Table.TableStatus == DynamoStatus.Active)
                        {
                            pendingTables.Remove(response.Table.TableName);
                        }
                    }

                    if (Log.IsDebugEnabled)
                    {
                        Log.Debug($"Tables Pending: {pendingTables.ToJsv()}");
                    }

                    if (pendingTables.Count == 0)
                    {
                        return(true);
                    }

                    if (timeout != null && DateTime.UtcNow - startAt > timeout.Value)
                    {
                        return(false);
                    }

                    Thread.Sleep(PollTableStatus);
                }
                catch (ResourceNotFoundException)
                {
                    // DescribeTable is eventually consistent. So you might
                    // get resource not found. So we handle the potential exception.
                }
            } while (true);
        }
示例#29
0
        public T PutItem <T>(T value, bool returnOld = false)
        {
            var table   = DynamoMetadata.GetTable <T>();
            var request = new PutItemRequest
            {
                TableName    = table.Name,
                Item         = Converters.ToAttributeValues(this, value, table),
                ReturnValues = returnOld ? ReturnValue.ALL_OLD : ReturnValue.NONE,
            };

            var response = Exec(() => DynamoDb.PutItem(request));

            if (response.Attributes.IsEmpty())
            {
                return(default);
示例#30
0
        private static WatchmanConfiguration Config()
        {
            var allTablesReadOnly = new DynamoDb
            {
                MonitorThrottling = true,
                Tables            = new List <Table>
                {
                    new Table {
                        Pattern = ".*", Threshold = 0.75, MonitorWrites = false
                    }
                }
            };

            return(AlertingGroupData.WrapDynamo(allTablesReadOnly));
        }