private static async Task <BatchWriteItemResponse> BuildDelete <Item>(Item[] items, string tableName, ItemConverter <Item> itemConverter, AmazonDynamoDBClient dynamodbClient) { List <WriteRequest> writeRequests = new List <WriteRequest>(); foreach (Item item in items) { DeleteRequest deleteRequest = new DeleteRequest() { Key = itemConverter.GetItemAsAttributes(item) }; WriteRequest writeRequest = new WriteRequest() { DeleteRequest = deleteRequest }; writeRequests.Add(writeRequest); } BatchWriteItemRequest batchWriteItemRequest = new BatchWriteItemRequest() { RequestItems = new Dictionary <string, List <WriteRequest> >() { { tableName, writeRequests } } }; return(await dynamodbClient.BatchWriteItemAsync(batchWriteItemRequest)); }
// This processes 25 quotes at a time since the max is 25 as per dynamoDB batch write private async Task WriteBatch(IEnumerable <Quote> batchQuotes) { // There is this possibility when there is nothing to approve if (batchQuotes.Count() == 0) { Console.WriteLine("Batch empty"); return; } Console.WriteLine("Writing batch"); Dictionary <string, List <WriteRequest> > writeRequests = new Dictionary <string, List <WriteRequest> >(); writeRequests.Add(DataDefinitions.QUOTES_TABLE, new List <WriteRequest>()); foreach (Quote quote in batchQuotes) { PutRequest request = new PutRequest(QuoteExtensions.AttributesOfQuote(quote)); writeRequests[DataDefinitions.QUOTES_TABLE].Add(new WriteRequest(request)); } BatchWriteItemRequest batchRequest = new BatchWriteItemRequest { RequestItems = writeRequests }; Console.WriteLine("Before writing to dynamodb"); BatchWriteItemResponse batchResponse = await _client.BatchWriteItemAsync(batchRequest); Console.WriteLine($"quotes batch response: {batchResponse.HttpStatusCode}"); }
/// <summary> /// Delete multiple entries from a DynamoDB table (Batch delete) /// </summary> /// <param name="tableName">The name of the table to delete entries</param> /// <param name="toDelete">List of key values for each entry that must be deleted in the batch</param> /// <returns></returns> public Task DeleteEntriesAsync(string tableName, IReadOnlyCollection <Dictionary <string, AttributeValue> > toDelete) { if (Logger.IsVerbose2) { Logger.Verbose2("Deleting {0} table entries", tableName); } if (toDelete == null) { throw new ArgumentNullException("collection"); } if (toDelete.Count == 0) { return(TaskDone.Done); } try { var request = new BatchWriteItemRequest(); request.RequestItems = new Dictionary <string, List <WriteRequest> >(); var batch = new List <WriteRequest>(); foreach (var keys in toDelete) { var writeRequest = new WriteRequest(); writeRequest.DeleteRequest = new DeleteRequest(); writeRequest.DeleteRequest.Key = keys; batch.Add(writeRequest); } request.RequestItems.Add(tableName, batch); return(ddbClient.BatchWriteItemAsync(request)); } catch (Exception exc) { Logger.Warn(ErrorCode.StorageProviderBase, $"Intermediate error deleting entries from the table {tableName}.", exc); throw; } }
private async Task WriteSeries(int?rowCount, bool runContinuous, bool useGroupedTable, string seriesKey) { using (var client = new AmazonDynamoDBClient(credentials, config)) { var dataSet = new DataSet(seriesKey); var writeToTableName = useGroupedTable ? groupedTableName : tableName; var records = useGroupedTable ? dataSet.GetRowGroups(rowCount, runContinuous).Select(CreateWriteRequest) : dataSet.GetRows(rowCount, runContinuous).Select(CreateWriteRequest); var count = 0; foreach (var batch in records.Batch(25)) // DynamoDB supports writing 25 records at a time { var requestItems = new Dictionary <string, List <WriteRequest> > { [writeToTableName] = batch }; while (requestItems.Any()) { var request = new BatchWriteItemRequest() { RequestItems = requestItems }; try { var response = await client.BatchWriteItemAsync(request); // Attempt to write unprocessed items again requestItems = response.UnprocessedItems; } catch (AmazonDynamoDBException ex) { await output.WriteLineAsync(ex.Message); } } count += batch.Count; Interlocked.Add(ref totalCount, batch.Count); await output.WriteLineAsync($"Wrote {count} records from dataset '{dataSet.SeriesKey}' to: {writeToTableName}"); } } }
/// <summary> /// Sends a list of write requests in batches as large as the AWS SDK will allow. /// </summary> /// <param name="client">the client</param> /// <param name="tableName">the table name</param> /// <param name="requests">list of requests</param> /// <returns>async Task with no return value</returns> public static async Task BatchWriteRequestsAsync(AmazonDynamoDBClient client, string tableName, List <WriteRequest> requests) { int batchSize = 25; for (int i = 0; i < requests.Count; i += batchSize) { var batch = requests.GetRange(i, Math.Min(batchSize, requests.Count - i)); var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> >() { { tableName, batch } }); await client.BatchWriteItemAsync(request); } }
private static async void CallBatchWriteTillCompletion(AmazonDynamoDBClient client, BatchWriteItemRequest request) { BatchWriteItemResponse response; int callCount = 0; do { Console.WriteLine("Making request"); response = await client.BatchWriteItemAsync(request); callCount++; // Check the response. var tableConsumedCapacities = response.ConsumedCapacity; var unprocessed = response.UnprocessedItems; Console.WriteLine("Per-table consumed capacity"); foreach (var tableConsumedCapacity in tableConsumedCapacities) { Console.WriteLine("{0} - {1}", tableConsumedCapacity.TableName, tableConsumedCapacity.CapacityUnits); } Console.WriteLine("Unprocessed"); foreach (var unp in unprocessed) { Console.WriteLine("{0} - {1}", unp.Key, unp.Value.Count); } Console.WriteLine(); // For the next iteration, the request will have unprocessed items. request.RequestItems = unprocessed; } while (response.UnprocessedItems.Count > 0); Console.WriteLine("Total # of batch write API calls made: {0}", callCount); }
public async Task ClearAsync(CancellationToken cancellationToken) { Dictionary <string, AttributeValue> lastKeyEvaluated = null; do { var request = new QueryRequest { TableName = _tableName, KeyConditionExpression = "Dictionary = :v_Dictionary", ExpressionAttributeValues = new Dictionary <string, AttributeValue> { { ":v_Dictionary", new AttributeValue(DictionaryName) }, }, ExpressionAttributeNames = new Dictionary <string, string>() { { "#K", "Key" } // MEMO: 'Key' is reserved keyword in DynamoDB Query. }, ProjectionExpression = "Dictionary, #K", }; var response = await _dynamoDBClient.QueryAsync(request, cancellationToken).ConfigureAwait(false); if (cancellationToken.IsCancellationRequested) { return; } if (!response.Items.Any()) { return; } var batchRequest = new BatchWriteItemRequest { RequestItems = new Dictionary <string, List <WriteRequest> > { { _tableName, response.Items.Select(x => new WriteRequest { DeleteRequest = new DeleteRequest { Key = new Dictionary <string, AttributeValue> { { "Dictionary", x["Dictionary"] }, { "Key", x["Key"] }, } } }).ToList() } } }; await _dynamoDBClient.BatchWriteItemAsync(batchRequest, cancellationToken).ConfigureAwait(false); if (cancellationToken.IsCancellationRequested) { return; } lastKeyEvaluated = response.LastEvaluatedKey; } while (lastKeyEvaluated != null && lastKeyEvaluated.Count != 0); }
/// <summary> /// Example that writes to a DynamoDb more than one item in one call. /// </summary> /// <returns></returns> public static async Task BatchWriteExampleAsync() { //Create Client var client = new AmazonDynamoDBClient(RegionEndpoint.USWest2); //Definition of the first item to put on a table var putFirstItem = new PutRequest(new Dictionary <string, AttributeValue> { { "pk", new AttributeValue("*****@*****.**") }, { "sk", new AttributeValue("metadata") }, { "attribute1", new AttributeValue("Attribute1 value") } //Add other attributes as you need } ); //Definition of the second item to put on a table var putSecondItem = new PutRequest(new Dictionary <string, AttributeValue> { { "pk", new AttributeValue("*****@*****.**") }, { "sk", new AttributeValue("metadata") }, { "attribute1", new AttributeValue("Attribute1 value") } //Add other attributes as you need } ); //Definition of an item to delete var deleteItem = new DeleteRequest(new Dictionary <string, AttributeValue> { { "pk", new AttributeValue("*****@*****.**") }, { "sk", new AttributeValue("metadata") }, } ); //Request that group all the previous Put & Delete actions var writeRequest = new BatchWriteItemRequest { RequestItems = new Dictionary <string, List <WriteRequest> > { { //Name of the table "RetailDatabase", new List <WriteRequest>() { new WriteRequest(putFirstItem), new WriteRequest(putSecondItem), new WriteRequest(deleteItem) } },//You can execute other collections of requests on other tables at the same time { //Name of the table "RetailDatabase2", new List <WriteRequest>() { new WriteRequest(putFirstItem), new WriteRequest(putSecondItem), new WriteRequest(deleteItem) } } } }; try { //Execution of the request var responseWrite = await client.BatchWriteItemAsync(writeRequest); Console.WriteLine($"Status {responseWrite.HttpStatusCode}"); Console.WriteLine($"Number of items not processed that you need to try again:{responseWrite.UnprocessedItems}"); } catch (Exception e) { Console.Error.WriteLine(e.Message); } }