private T ConvertGetItemResponse <T>(GetItemRequest request, DynamoMetadataType table)
{
var response = Exec(() => DynamoDb.GetItem(request), rethrowExceptions: throwNotFoundExceptions);
if (!response.IsItemSet)
{
return(default);
public long DescribeItemCount <T>()
{
var table = DynamoMetadata.GetTable <T>();
var response = Exec(() => DynamoDb.DescribeTable(new DescribeTableRequest(table.Name)));
return(response.Table.ItemCount);
}
private async Task <List <Table> > ExpandTablePatterns(DynamoDb dynamoDb, string alertingGroupName)
{
var tablesWithoutPatterns = dynamoDb.Tables
.Where(t => string.IsNullOrWhiteSpace(t.Pattern))
.ToList();
var patterns = dynamoDb.Tables
.Where(t => !string.IsNullOrWhiteSpace(t.Pattern))
.ToList();
var tablesFromPatterns = new List <Table>();
foreach (var tablePattern in patterns)
{
var matches = await GetPatternMatches(tablePattern, alertingGroupName);
// filter out duplicates
matches = matches
.Where(match => tablesFromPatterns.All(t => t.Name != match.Name))
.ToList();
matches = matches
.Where(match => tablesWithoutPatterns.All(t => t.Name != match.Name))
.ToList();
tablesFromPatterns.AddRange(matches);
}
return(tablesWithoutPatterns
.Union(tablesFromPatterns)
.ToList());
}
public async Task UpdateItemAsync <T>(DynamoUpdateItem update, CancellationToken token = default)
{
var request = ToUpdateItemRequest <T>(update);
await ExecAsync(async() =>
await DynamoDb.UpdateItemAsync(request, token).ConfigAwait()).ConfigAwait();
}
public void PutItems <T>(IEnumerable <T> items)
{
var table = DynamoMetadata.GetTable <T>();
var remaining = items.ToList();
PopulateMissingHashes(table, remaining);
while (remaining.Count > 0)
{
var batchSize = Math.Min(remaining.Count, MaxWriteBatchSize);
var nextBatch = remaining.GetRange(0, batchSize);
remaining.RemoveRange(0, batchSize);
var putItems = nextBatch.Map(x => new WriteRequest(
new PutRequest(Converters.ToAttributeValues(this, x, table))));
var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> > {
{ table.Name, putItems }
});
var response = Exec(() => DynamoDb.BatchWriteItem(request));
var i = 0;
while (response.UnprocessedItems.Count > 0)
{
response = Exec(() => DynamoDb.BatchWriteItem(new BatchWriteItemRequest(response.UnprocessedItems)));
if (response.UnprocessedItems.Count > 0)
{
i.SleepBackOffMultiplier();
}
}
}
}
private List <T> ConvertBatchGetItemResponse <T>(DynamoMetadataType table, KeysAndAttributes getItems)
{
var to = new List <T>();
var request = new BatchGetItemRequest(new Dictionary <string, KeysAndAttributes> {
{ table.Name, getItems }
});
var response = Exec(() => DynamoDb.BatchGetItem(request));
List <Dictionary <string, AttributeValue> > results;
if (response.Responses.TryGetValue(table.Name, out results))
{
results.Each(x => to.Add(Converters.FromAttributeValues <T>(table, x)));
}
var i = 0;
while (response.UnprocessedKeys.Count > 0)
{
response = Exec(() => DynamoDb.BatchGetItem(new BatchGetItemRequest(response.UnprocessedKeys)));
if (response.Responses.TryGetValue(table.Name, out results))
{
results.Each(x => to.Add(Converters.FromAttributeValues <T>(table, x)));
}
if (response.UnprocessedKeys.Count > 0)
{
i.SleepBackOffMultiplier();
}
}
return(to);
}
public long Increment <T>(object hash, string fieldName, long amount = 1)
{
var type = DynamoMetadata.GetType <T>();
var request = new UpdateItemRequest
{
TableName = type.Name,
Key = Converters.ToAttributeKeyValue(this, type.HashKey, hash),
AttributeUpdates = new Dictionary <string, AttributeValueUpdate> {
{
fieldName,
new AttributeValueUpdate {
Action = AttributeAction.ADD,
Value = new AttributeValue {
N = amount.ToString()
}
}
}
},
ReturnValues = ReturnValue.ALL_NEW,
};
var response = DynamoDb.UpdateItem(request);
return(response.Attributes.Count > 0
? Convert.ToInt64(response.Attributes[fieldName].N)
: 0);
}
public List <T> Scan <T>(ScanRequest request, int limit)
{
var to = new List <T>();
if (request.Limit == default(int))
{
request.Limit = limit;
}
ScanResponse response = null;
do
{
if (response != null)
{
request.ExclusiveStartKey = response.LastEvaluatedKey;
}
response = Exec(() => DynamoDb.Scan(request));
var results = response.ConvertAll <T>();
foreach (var result in results)
{
to.Add(result);
if (to.Count >= limit)
{
break;
}
}
} while (!response.LastEvaluatedKey.IsEmpty() && to.Count < limit);
return(to);
}
private async Task <bool> TableExist()
{
var tables = await DynamoDb.ListTablesAsync();
var existTable = tables.TableNames.Contains(TableName);
return(existTable);
}
public async Task <bool> TableExist(string tableName)
{
var tables = await DynamoDb.ListTablesAsync();
var existTable = tables.TableNames.Contains(tableName);
return(existTable);
}
protected virtual void Dispose(bool disposing)
{
if (DynamoDb != null)
{
DynamoDb.Dispose();
DynamoDb = null;
}
}
public void ClearEntities()
{
//TODO :DI
using (IDal database = new DynamoDb())
{
database.ClearEntities();
}
}
public void UpdateItem <T>(DynamoUpdateItem update)
{
var table = DynamoMetadata.GetTable <T>();
var request = new UpdateItemRequest
{
TableName = table.Name,
Key = Converters.ToAttributeKeyValue(this, table, update.Hash, update.Range),
AttributeUpdates = new Dictionary <string, AttributeValueUpdate>(),
ReturnValues = ReturnValue.NONE,
};
if (update.Put != null)
{
foreach (var entry in update.Put)
{
var field = table.GetField(entry.Key);
if (field == null)
{
continue;
}
request.AttributeUpdates[field.Name] = new AttributeValueUpdate(
Converters.ToAttributeValue(this, field.Type, field.DbType, entry.Value), DynamoAttributeAction.Put);
}
}
if (update.Add != null)
{
foreach (var entry in update.Add)
{
var field = table.GetField(entry.Key);
if (field == null)
{
continue;
}
request.AttributeUpdates[field.Name] = new AttributeValueUpdate(
Converters.ToAttributeValue(this, field.Type, field.DbType, entry.Value), DynamoAttributeAction.Add);
}
}
if (update.Delete != null)
{
foreach (var key in update.Delete)
{
var field = table.GetField(key);
if (field == null)
{
continue;
}
request.AttributeUpdates[field.Name] = new AttributeValueUpdate(null, DynamoAttributeAction.Delete);
}
}
Exec(() => DynamoDb.UpdateItem(request));
}
private async Task CreateIfNotExist(CreateTableRequest request)
{
if (await TableExist())
{
return;
}
await DynamoDb.CreateTableAsync(request);
}
protected Task <CreateTableResponse> CreateTestTable(string tableName = null)
{
var postTableCreateRequest = new CreateTableRequest
{
AttributeDefinitions =
new List <AttributeDefinition>
{
new AttributeDefinition {
AttributeName = nameof(MovieEntity.DirectorId), AttributeType = ScalarAttributeType.S
},
new AttributeDefinition {
AttributeName = nameof(MovieEntity.CreateDate), AttributeType = ScalarAttributeType.S
},
new AttributeDefinition()
{
AttributeName = nameof(MovieEntity.MovieId), AttributeType = ScalarAttributeType.S
}
},
TableName = tableName ?? TestTableName,
KeySchema =
new List <KeySchemaElement>()
{
new KeySchemaElement()
{
AttributeName = nameof(MovieEntity.DirectorId), KeyType = KeyType.HASH
},
new KeySchemaElement()
{
AttributeName = nameof(MovieEntity.CreateDate), KeyType = KeyType.RANGE
}
},
GlobalSecondaryIndexes = new List <GlobalSecondaryIndex>
{
new GlobalSecondaryIndex
{
Projection = new Projection {
ProjectionType = ProjectionType.ALL
},
IndexName = TestConstants.MoiveTableMovieIdGsi,
KeySchema = new List <KeySchemaElement> {
new KeySchemaElement {
AttributeName = nameof(MovieEntity.MovieId), KeyType = KeyType.HASH
}
},
ProvisionedThroughput = new ProvisionedThroughput {
ReadCapacityUnits = 5, WriteCapacityUnits = 5
}
}
},
ProvisionedThroughput = new ProvisionedThroughput {
ReadCapacityUnits = 5, WriteCapacityUnits = 6
},
};
return(DynamoDb.CreateTableAsync(postTableCreateRequest));
}
public void Close()
{
if (DynamoDb == null)
{
return;
}
DynamoDb.Dispose();
DynamoDb = null;
}
public T Get(long Id)
{
try
{
T item = DynamoDb.GetItem <T>(Id);
return(item);
}
catch {
return(null);
}
}
public List <T> GetAll()
{
try
{
List <T> Items = DynamoDb.GetAll <T>();
return(Items);
}
catch (Exception ex) {
return(new List <T>());
}
}
public bool SaveToDatabase(IList <Entity> entities)
{
bool result;
//TODO :DI
using (IDal database = new DynamoDb())
{
result = database.InsertEntities(entities);
}
return(result);
}
private T ConvertGetItemResponse <T>(GetItemRequest request, DynamoMetadataType table)
{
var response = Exec(() => DynamoDb.GetItem(request), rethrowExceptions: throwNotFoundExceptions);
if (!response.IsItemSet)
{
return(default(T));
}
var attributeValues = response.Item;
return(Converters.FromAttributeValues <T>(table, attributeValues));
}
public bool UpdateItem <T>(UpdateExpression <T> update)
{
try
{
Exec(() => DynamoDb.UpdateItem(update));
return(true);
}
catch (ConditionalCheckFailedException)
{
return(false);
}
}
public Entity SearchEntity(Entity entity)
{
//TODO :DI
Entity result;
Enum.TryParse(entity.Type, out EntityType type);
using (IDal database = new DynamoDb())
{
result = database.QueryEntity(type, entity.Id, entity.TimeStamp);
}
return(result);
}
public async Task <bool> UpdateItemAsync <T>(UpdateExpression <T> update, CancellationToken token = default)
{
try
{
await ExecAsync(async() =>
await DynamoDb.UpdateItemAsync(update, token).ConfigAwait()).ConfigAwait();
return(true);
}
catch (ConditionalCheckFailedException)
{
return(false);
}
}
private static WatchmanConfiguration Config()
{
var group = new DynamoDb
{
Tables = new List <Table>
{
new Table {
Pattern = ".*", Threshold = 0.75, MonitorWrites = false
}
}
};
return(AlertingGroupData.WrapDynamo(group));
}
private static WatchmanConfiguration Config()
{
var ag = new DynamoDb
{
Tables = new List <Table>
{
new Table {
Name = "authentication-tokens"
}
}
};
return(AlertingGroupData.WrapDynamo(ag));
}
private static WatchmanConfiguration CatchAllConfig()
{
var catchAll = new DynamoDb
{
Tables = new List <Table>
{
new Table {
Pattern = ".*"
}
}
};
return(AlertingGroupData.WrapDynamo(catchAll));
}
private static WatchmanConfiguration Config()
{
var dynamoConfig = new DynamoDb
{
Tables = new List <Table>
{
new Table {
Pattern = "nomatchwillbefoundforthis"
}
}
};
return(AlertingGroupData.WrapDynamo(dynamoConfig));
}
public bool WaitForTablesToBeReady(IEnumerable <string> tableNames, TimeSpan?timeout = null)
{
var pendingTables = new List <string>(tableNames);
if (pendingTables.Count == 0)
{
return(true);
}
var startAt = DateTime.UtcNow;
do
{
try
{
var responses = pendingTables.Map(x =>
Exec(() => DynamoDb.DescribeTable(new DescribeTableRequest(x))));
foreach (var response in responses)
{
if (response.Table.TableStatus == DynamoStatus.Active)
{
pendingTables.Remove(response.Table.TableName);
}
}
if (Log.IsDebugEnabled)
{
Log.Debug($"Tables Pending: {pendingTables.ToJsv()}");
}
if (pendingTables.Count == 0)
{
return(true);
}
if (timeout != null && DateTime.UtcNow - startAt > timeout.Value)
{
return(false);
}
Thread.Sleep(PollTableStatus);
}
catch (ResourceNotFoundException)
{
// DescribeTable is eventually consistent. So you might
// get resource not found. So we handle the potential exception.
}
} while (true);
}
public T PutItem <T>(T value, bool returnOld = false)
{
var table = DynamoMetadata.GetTable <T>();
var request = new PutItemRequest
{
TableName = table.Name,
Item = Converters.ToAttributeValues(this, value, table),
ReturnValues = returnOld ? ReturnValue.ALL_OLD : ReturnValue.NONE,
};
var response = Exec(() => DynamoDb.PutItem(request));
if (response.Attributes.IsEmpty())
{
return(default);
private static WatchmanConfiguration Config()
{
var allTablesReadOnly = new DynamoDb
{
MonitorThrottling = true,
Tables = new List <Table>
{
new Table {
Pattern = ".*", Threshold = 0.75, MonitorWrites = false
}
}
};
return(AlertingGroupData.WrapDynamo(allTablesReadOnly));
}