public static async Task ClearAllData(string prefix) { var keyPrefix = prefix is null ? "" : (prefix + ":"); using (var client = CreateTestClient()) { var deleteReqs = new List <WriteRequest>(); ScanRequest request = new ScanRequest(TableName) { ConsistentRead = true, ProjectionExpression = "#namespace, #key", ExpressionAttributeNames = new Dictionary <string, string>() { { "#namespace", DynamoDB.DataStorePartitionKey }, { "#key", DynamoDB.DataStoreSortKey } } }; await DynamoDBHelpers.IterateScan(client, request, item => { if (item[DynamoDB.DataStorePartitionKey].S.StartsWith(keyPrefix)) { deleteReqs.Add(new WriteRequest(new DeleteRequest(item))); } }); await DynamoDBHelpers.BatchWriteRequestsAsync(client, TableName, deleteReqs); } }
public async Task InitAsync(FullDataSet <SerializedItemDescriptor> allData) { // Start by reading the existing keys; we will later delete any of these that weren't in allData. var unusedOldKeys = await ReadExistingKeys(allData.Data.Select(collection => collection.Key)); var requests = new List <WriteRequest>(); var numItems = 0; // Insert or update every provided item foreach (var collection in allData.Data) { var kind = collection.Key; foreach (var keyAndItem in collection.Value.Items) { var encodedItem = MarshalItem(kind, keyAndItem.Key, keyAndItem.Value); if (!CheckSizeLimit(encodedItem)) { continue; } requests.Add(new WriteRequest(new PutRequest(encodedItem))); var combinedKey = new Tuple <string, string>(NamespaceForKind(kind), keyAndItem.Key); unusedOldKeys.Remove(combinedKey); numItems++; } } // Now delete any previously existing items whose keys were not in the current data foreach (var combinedKey in unusedOldKeys) { if (combinedKey.Item1 != InitedKey) { var keys = MakeKeysMap(combinedKey.Item1, combinedKey.Item2); requests.Add(new WriteRequest(new DeleteRequest(keys))); } } // Now set the special key that we check in initializedInternal() var initedItem = MakeKeysMap(InitedKey, InitedKey); requests.Add(new WriteRequest(new PutRequest(initedItem))); await DynamoDBHelpers.BatchWriteRequestsAsync(_client, _tableName, requests); _log.Info("Initialized data store with {0} items", numItems); }
public async Task <KeyedItems <SerializedItemDescriptor> > GetAllAsync(DataKind kind) { var ret = new List <KeyValuePair <string, SerializedItemDescriptor> >(); var req = MakeQueryForKind(kind); await DynamoDBHelpers.IterateQuery(_client, req, item => { var itemOut = UnmarshalItem(kind, item); if (itemOut.HasValue) { var itemKey = item[DynamoDB.DataStoreSortKey].S; ret.Add(new KeyValuePair <string, SerializedItemDescriptor>(itemKey, itemOut.Value)); } }); return(new KeyedItems <SerializedItemDescriptor>(ret)); }
private async Task <HashSet <Tuple <string, string> > > ReadExistingKeys(IEnumerable <DataKind> kinds) { var keys = new HashSet <Tuple <string, string> >(); foreach (var kind in kinds) { var req = MakeQueryForKind(kind); req.ProjectionExpression = "#namespace, #key"; req.ExpressionAttributeNames = new Dictionary <string, string>() { { "#namespace", DynamoDB.DataStorePartitionKey }, { "#key", DynamoDB.DataStoreSortKey } }; await DynamoDBHelpers.IterateQuery(_client, req, item => keys.Add(new Tuple <string, string>( item[DynamoDB.DataStorePartitionKey].S, item[DynamoDB.DataStoreSortKey].S)) ); } return(keys); }