private static DynamoMetadataType ToMetadataType(Type type)
{
var alias = type.FirstAttribute <AliasAttribute>();
var props = GetTableProperties(type);
var metadata = new DynamoMetadataType
{
Type = type,
Name = alias != null ? alias.Name : type.Name,
};
metadata.Fields = props.Map(p =>
new DynamoMetadataField
{
Parent = metadata,
Type = p.PropertyType,
Name = Converters.GetFieldName(p),
DbType = Converters.GetFieldType(p.PropertyType),
IsAutoIncrement = p.HasAttribute <AutoIncrementAttribute>(),
SetValueFn = p.CreateSetter(),
GetValueFn = p.CreateGetter(),
}).ToArray();
return(metadata);
}
private List <T> ConvertBatchGetItemResponse <T>(DynamoMetadataType table, KeysAndAttributes getItems)
{
var to = new List <T>();
var request = new BatchGetItemRequest(new Dictionary <string, KeysAndAttributes> {
{ table.Name, getItems }
});
var response = Exec(() => DynamoDb.BatchGetItem(request));
if (response.Responses.TryGetValue(table.Name, out var results))
{
results.Each(x => to.Add(Converters.FromAttributeValues <T>(table, x)));
}
var i = 0;
while (response.UnprocessedKeys.Count > 0)
{
response = Exec(() => DynamoDb.BatchGetItem(new BatchGetItemRequest(response.UnprocessedKeys)));
if (response.Responses.TryGetValue(table.Name, out results))
{
results.Each(x => to.Add(Converters.FromAttributeValues <T>(table, x)));
}
if (response.UnprocessedKeys.Count > 0)
{
i.SleepBackOffMultiplier();
}
}
return(to);
}
private static DynamoLocalIndex CreateLocalIndex(Type type, DynamoMetadataType metadata, string hashField, Type indexType)
{
var indexProps = indexType.GetPublicProperties();
var indexProp = indexProps.FirstOrDefault(x =>
x.HasAttribute <IndexAttribute>() || x.HasAttribute <DynamoDBRangeKeyAttribute>());
if (indexProp == null)
{
throw new ArgumentException($"Missing [Index]. Could not infer Range Key in index '{indexType}'.");
}
var indexAlias = indexType.FirstAttribute <AliasAttribute>();
var rangeKey = metadata.GetField(indexProp.Name);
if (rangeKey == null)
{
throw new ArgumentException($"Range Key '{indexProp.Name}' was not found on Table '{type.Name}'");
}
var indexProjection = indexType.FirstAttribute <ProjectionTypeAttribute>();
var projectionType = indexProjection?.ProjectionType ?? DynamoProjectionType.Include;
return(new DynamoLocalIndex
{
IndexType = indexType,
Name = indexAlias != null ? indexAlias.Name : indexType.Name,
HashKey = metadata.HashKey,
RangeKey = rangeKey,
ProjectionType = projectionType,
ProjectedFields = projectionType == DynamoProjectionType.Include
? indexProps.Where(x => x.Name != hashField).Select(x => x.Name).ToArray()
: new string[0],
});
}
private void PopulateMissingHashes <T>(DynamoMetadataType table, List <T> items)
{
var autoIncr = table.Fields.FirstOrDefault(x => x.IsAutoIncrement);
if (autoIncr != null)
{
var seqRequiredPos = new List <int>();
for (int i = 0; i < items.Count; i++)
{
var item = items[i];
var value = autoIncr.GetValue(item);
if (DynamoConverters.IsNumberDefault(value))
{
seqRequiredPos.Add(i);
}
}
if (seqRequiredPos.Count == 0)
{
return;
}
var nextSequences = Sequences.GetNextSequences(table, seqRequiredPos.Count);
for (int i = 0; i < nextSequences.Length; i++)
{
var pos = seqRequiredPos[i];
autoIncr.SetValue(items[pos], nextSequences[i]);
}
}
}
public DynamoDbQueryDataSource(QueryDataContext context, IPocoDynamo dynamo, bool allowScans = true)
: base(context)
{
this.db = dynamo;
this.modelDef = db.GetTableMetadata <T>();
isGlobalIndex = typeof(T).IsGlobalIndex();
this.allowScans = allowScans;
}
private T ConvertGetItemResponse <T>(GetItemRequest request, DynamoMetadataType table)
{
var response = Exec(() => DynamoDb.GetItem(request), rethrowExceptions: throwNotFoundExceptions);
if (!response.IsItemSet)
{
return(default(T));
}
var attributeValues = response.Item;
return(Converters.FromAttributeValues <T>(table, attributeValues));
}
private static object ApplyFieldBehavior(IPocoDynamo db, DynamoMetadataType type, DynamoMetadataField field, object instance, object value)
{
if (type == null || field == null || !field.IsAutoIncrement)
{
return(value);
}
var needsId = IsNumberDefault(value);
if (!needsId)
{
return(value);
}
var nextId = db.Sequences.Increment(type.Name);
return(field.SetValue(instance, nextId));
}
private Task CreateTableAsync(DynamoMetadataType table)
{
var request = ToCreateTableRequest(table);
try
{
return(ExecAsync(() => DynamoDb.CreateTableAsync(request)));
}
catch (AmazonDynamoDBException ex)
{
if (ex.ErrorCode == DynamoErrors.AlreadyExists)
{
return(PclExportClient.EmptyTask);
}
throw;
}
}
private static DynamoGlobalIndex CreateGlobalIndex(Type type, DynamoMetadataType metadata, Type indexType)
{
var indexProps = indexType.GetPublicProperties();
Converters.GetHashAndRangeKeyFields(indexType, indexProps, out var indexHash, out var indexRange);
var hashKey = metadata.GetField(indexHash.Name);
if (hashKey == null)
{
throw new ArgumentException($"Hash Key '{indexHash.Name}' was not found on Table '{type.Name}'");
}
if (indexRange == null)
{
indexRange = indexProps.FirstOrDefault(x => x.HasAttribute <IndexAttribute>());
}
var rangeKey = indexRange != null
? metadata.GetField(indexRange.Name)
: null;
var indexAlias = indexType.FirstAttribute <AliasAttribute>();
var indexProvision = indexType.FirstAttribute <ProvisionedThroughputAttribute>();
var indexProjection = indexType.FirstAttribute <ProjectionTypeAttribute>();
var projectionType = indexProjection?.ProjectionType ?? DynamoProjectionType.Include;
return(new DynamoGlobalIndex
{
IndexType = indexType,
Name = indexAlias != null ? indexAlias.Name : indexType.Name,
HashKey = hashKey,
RangeKey = rangeKey,
ProjectionType = projectionType,
ProjectedFields = projectionType == DynamoProjectionType.Include
? indexProps.Where(x => x.Name != indexHash.Name).Select(x => x.Name).ToArray()
: new string[0],
ReadCapacityUnits = indexProvision?.ReadCapacityUnits ?? metadata.ReadCapacityUnits,
WriteCapacityUnits = indexProvision?.WriteCapacityUnits ?? metadata.WriteCapacityUnits,
});
}
private void ExecBatchWriteItemResponse <T>(DynamoMetadataType table, List <WriteRequest> deleteItems)
{
var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> >
{
{ table.Name, deleteItems }
});
var response = Exec(() => DynamoDb.BatchWriteItem(request));
var i = 0;
while (response.UnprocessedItems.Count > 0)
{
response = Exec(() => DynamoDb.BatchWriteItem(new BatchWriteItemRequest(response.UnprocessedItems)));
if (response.UnprocessedItems.Count > 0)
{
i.SleepBackOffMultiplier();
}
}
}
private void CreateTable(DynamoMetadataType table)
{
var request = ToCreateTableRequest(table);
Exec(() =>
{
try
{
DynamoDb.CreateTable(request);
}
catch (AmazonDynamoDBException ex)
{
if (ex.ErrorCode == DynamoErrors.AlreadyExists ||
ex.Message == "Cannot create preexisting table")
{
return;
}
throw;
}
});
}
public static T ConvertTo <T>(this DynamoMetadataType table,
Dictionary <string, AttributeValue> attributeValues)
{
return(DynamoMetadata.Converters.FromAttributeValues <T>(table, attributeValues));
}
public virtual Dictionary <string, AttributeValue> ToAttributeKeyValue(IPocoDynamo db, DynamoMetadataType table, DynamoId id)
{
using (AwsClientUtils.GetJsScope())
{
return(new Dictionary <string, AttributeValue> {
{ table.HashKey.Name, ToAttributeValue(db, table.HashKey.Type, table.HashKey.DbType, id.Hash) },
{ table.RangeKey.Name, ToAttributeValue(db, table.RangeKey.Type, table.RangeKey.DbType, id.Range) },
});
}
}
private static DynamoMetadataType ToMetadataTable(Type type)
{
var alias = type.FirstAttribute <AliasAttribute>();
var props = GetTableProperties(type);
Converters.GetHashAndRangeKeyFields(type, props, out var hash, out var range);
var provision = type.FirstAttribute <ProvisionedThroughputAttribute>();
// If its a generic type, the type name will contain illegal characters (not accepted as DynamoDB table name)
// so, remove the Tilde and make the type name unique to the runtime type of the generic
string genericTypeNameAlias = null;
if (type.IsGenericType)
{
var indexOfTilde = type.Name.IndexOf("`", StringComparison.Ordinal);
indexOfTilde = indexOfTilde < 1 ? type.Name.Length - 1 : indexOfTilde;
genericTypeNameAlias = type.Name.Substring(0, indexOfTilde) + type.GetGenericArguments().Select(t => t.Name).Join();
}
var metadata = new DynamoMetadataType
{
Type = type,
IsTable = true,
Name = alias != null ? alias.Name : genericTypeNameAlias ?? type.Name,
ReadCapacityUnits = provision?.ReadCapacityUnits,
WriteCapacityUnits = provision?.WriteCapacityUnits,
};
metadata.Fields = props.Map(p =>
new DynamoMetadataField
{
Parent = metadata,
Type = p.PropertyType,
Name = Converters.GetFieldName(p),
DbType = Converters.GetFieldType(p.PropertyType),
IsHashKey = p == hash,
IsRangeKey = p == range,
ExcludeNullValue = p.HasAttribute <IndexAttribute>() || p.HasAttribute <ExcludeNullValueAttribute>(),
IsAutoIncrement = p.HasAttribute <AutoIncrementAttribute>(),
SetValueFn = p.CreateSetter(),
GetValueFn = p.CreateGetter(),
}).ToArray();
metadata.HashKey = metadata.Fields.FirstOrDefault(x => x.IsHashKey);
metadata.RangeKey = metadata.Fields.FirstOrDefault(x => x.IsRangeKey);
if (metadata.HashKey == null)
{
throw new ArgumentException($"Could not infer Hash Key in Table '{type.Name}'");
}
var hashField = metadata.HashKey.Name;
string indexName = null;
metadata.LocalIndexes = props.Where(x => x.HasAttribute <IndexAttribute>()).Map(x =>
{
var indexProjection = x.FirstAttribute <ProjectionTypeAttribute>();
var projectionType = indexProjection?.ProjectionType ?? DynamoProjectionType.Include;
indexName = metadata.Name.ToIndexName(IndexType.Local, x.Name, metadata.HashKey.Name);
return(new DynamoLocalIndex
{
Name = indexName,
HashKey = metadata.HashKey,
RangeKey = metadata.GetField(x.Name),
ProjectionType = DynamoProjectionType.All,
ProjectedFields = new string[] { "" }
});
});
metadata.GlobalIndexes = props.Where(x => x.HasAttribute <GlobalSecondaryIndexHashKeyAttribute>()).Map(x =>
{
var indexProjection = x.FirstAttribute <ProjectionTypeAttribute>();
var globalSecondaryIndexHashKeyAttribute = x.GetCustomAttribute <GlobalSecondaryIndexHashKeyAttribute>();
var rangeKey = globalSecondaryIndexHashKeyAttribute.RangeKey;
bool hasRangeKey = !string.IsNullOrEmpty(rangeKey);
//string indexName = (hasRangeKey) ? $"{metadata.Name}-GSI-{x.Name}-{rangeKey}-Index" : $"{metadata.Name}-GSI-{x.Name}-Index";
indexName = metadata.Name.ToIndexName(IndexType.Global, x.Name, null, rangeKey);
var dynamoIndex = new DynamoGlobalIndex
{
Name = indexName,
HashKey = metadata.GetField(x.Name),
RangeKey = metadata.GetField(rangeKey),
ProjectionType = DynamoProjectionType.All,
ProjectedFields = new string[] { "" }
};
if (hasRangeKey)
{
dynamoIndex.RangeKey = metadata.GetField(rangeKey);
}
return(dynamoIndex);
});
var references = type.AllAttributes <ReferencesAttribute>();
foreach (var attr in references)
{
var localIndex = attr.Type.GetTypeWithGenericInterfaceOf(typeof(ILocalIndex <>));
if (localIndex != null)
{
metadata.LocalIndexes.Add(CreateLocalIndex(type, metadata, hashField, attr.Type));
}
var globalIndex = attr.Type.GetTypeWithGenericInterfaceOf(typeof(IGlobalIndex <>));
if (globalIndex != null)
{
metadata.GlobalIndexes.Add(CreateGlobalIndex(type, metadata, attr.Type));
}
}
return(metadata);
}
public virtual Dictionary <string, AttributeValue> ToAttributeKeyValue(IPocoDynamo db, DynamoMetadataType table, object hash, object range)
{
using (AwsClientUtils.GetJsScope())
{
var to = new Dictionary <string, AttributeValue> {
{ table.HashKey.Name, ToAttributeValue(db, table.HashKey.Type, table.HashKey.DbType, hash) },
};
if (range != null)
{
to[table.RangeKey.Name] = ToAttributeValue(db, table.RangeKey.Type, table.RangeKey.DbType, range);
}
return(to);
}
}
public virtual Dictionary <string, AttributeValue> ToAttributeKey(IPocoDynamo db, DynamoMetadataType table, object instance)
{
using (AwsClientUtils.GetJsScope())
{
var field = table.HashKey;
var to = new Dictionary <string, AttributeValue> {
{ field.Name, ToAttributeValue(db, field.Type, field.DbType, field.GetValue(instance)) },
};
if (table.RangeKey != null)
{
field = table.RangeKey;
to[field.Name] = ToAttributeValue(db, field.Type, field.DbType, field.GetValue(instance));
}
return(to);
}
}
public virtual Dictionary <string, AttributeValue> ToAttributeValues(IPocoDynamo db, object instance, DynamoMetadataType table)
{
var ret = ToAttributeValuesFn?.Invoke(instance, table);
if (ret != null)
{
return(ret);
}
using (AwsClientUtils.GetJsScope())
{
var to = new Dictionary <string, AttributeValue>();
foreach (var field in table.Fields)
{
var value = field.GetValue(instance);
value = ApplyFieldBehavior(db, table, field, instance, value);
if (value == null)
{
if (DynamoConfig.ExcludeNullValues || field.ExcludeNullValue)
{
continue;
}
}
to[field.Name] = ToAttributeValue(db, field.Type, field.DbType, value);
}
return(to);
}
}
public virtual Dictionary <string, AttributeValueUpdate> ToNonDefaultAttributeValueUpdates(IPocoDynamo db, object instance, DynamoMetadataType table)
{
using (AwsClientUtils.GetJsScope())
{
var to = new Dictionary <string, AttributeValueUpdate>();
foreach (var field in table.Fields)
{
if (field.IsHashKey || field.IsRangeKey)
{
continue;
}
var value = field.GetValue(instance);
if (value == null)
{
continue;
}
to[field.Name] = new AttributeValueUpdate(ToAttributeValue(db, field.Type, field.DbType, value), DynamoAttributeAction.Put);
}
return(to);
}
}
public static bool CreateTableIfMissing(this IPocoDynamo db, DynamoMetadataType table)
{
return(db.CreateMissingTables(new[] { table }));
}
protected virtual CreateTableRequest ToCreateTableRequest(DynamoMetadataType table)
{
var props = table.Type.GetSerializableProperties();
if (props.Length == 0)
{
throw new NotSupportedException($"{table.Name} does not have any serializable properties");
}
var keySchema = new List <KeySchemaElement> {
new KeySchemaElement(table.HashKey.Name, KeyType.HASH),
};
var attrDefinitions = new List <AttributeDefinition> {
new AttributeDefinition(table.HashKey.Name, table.HashKey.DbType),
};
if (table.RangeKey != null)
{
keySchema.Add(new KeySchemaElement(table.RangeKey.Name, KeyType.RANGE));
attrDefinitions.Add(new AttributeDefinition(table.RangeKey.Name, table.RangeKey.DbType));
}
var to = new CreateTableRequest
{
TableName = table.Name,
KeySchema = keySchema,
AttributeDefinitions = attrDefinitions,
ProvisionedThroughput = new ProvisionedThroughput
{
ReadCapacityUnits = table.ReadCapacityUnits ?? ReadCapacityUnits,
WriteCapacityUnits = table.WriteCapacityUnits ?? WriteCapacityUnits,
}
};
if (!table.LocalIndexes.IsEmpty())
{
to.LocalSecondaryIndexes = table.LocalIndexes.Map(x => new LocalSecondaryIndex
{
IndexName = x.Name,
KeySchema = x.ToKeySchemas(),
Projection = new Projection
{
ProjectionType = ProjectionType.ALL,
NonKeyAttributes = new List <string>()
},
});
table.LocalIndexes.Each(x =>
{
if (x.RangeKey != null && attrDefinitions.All(a => a.AttributeName != x.RangeKey.Name))
{
attrDefinitions.Add(new AttributeDefinition(x.RangeKey.Name, x.RangeKey.DbType));
}
});
}
if (!table.GlobalIndexes.IsEmpty())
{
to.GlobalSecondaryIndexes = table.GlobalIndexes.Map(x => new GlobalSecondaryIndex
{
IndexName = x.Name,
KeySchema = x.ToKeySchemas(),
Projection = new Projection
{
ProjectionType = ProjectionType.ALL,
NonKeyAttributes = new List <string>(),
},
ProvisionedThroughput = new ProvisionedThroughput
{
ReadCapacityUnits = x.ReadCapacityUnits ?? ReadCapacityUnits,
WriteCapacityUnits = x.WriteCapacityUnits ?? WriteCapacityUnits,
}
});
table.GlobalIndexes.Each(x =>
{
if (x.HashKey != null && attrDefinitions.All(a => a.AttributeName != x.HashKey.Name))
{
attrDefinitions.Add(new AttributeDefinition(x.HashKey.Name, x.HashKey.DbType));
}
if (x.RangeKey != null && attrDefinitions.All(a => a.AttributeName != x.RangeKey.Name))
{
attrDefinitions.Add(new AttributeDefinition(x.RangeKey.Name, x.RangeKey.DbType));
}
});
}
return(to);
}
