private static async Task <BatchWriteItemResponse> BuildDelete <Item>(Item[] items, string tableName, ItemConverter <Item> itemConverter, AmazonDynamoDBClient dynamodbClient)
{
List <WriteRequest> writeRequests = new List <WriteRequest>();
foreach (Item item in items)
{
DeleteRequest deleteRequest = new DeleteRequest()
{
Key = itemConverter.GetItemAsAttributes(item)
};
WriteRequest writeRequest = new WriteRequest()
{
DeleteRequest = deleteRequest
};
writeRequests.Add(writeRequest);
}
BatchWriteItemRequest batchWriteItemRequest = new BatchWriteItemRequest()
{
RequestItems = new Dictionary <string, List <WriteRequest> >()
{
{
tableName,
writeRequests
}
}
};
return(await dynamodbClient.BatchWriteItemAsync(batchWriteItemRequest));
}
// This processes 25 quotes at a time since the max is 25 as per dynamoDB batch write
private async Task WriteBatch(IEnumerable <Quote> batchQuotes)
{
// There is this possibility when there is nothing to approve
if (batchQuotes.Count() == 0)
{
Console.WriteLine("Batch empty");
return;
}
Console.WriteLine("Writing batch");
Dictionary <string, List <WriteRequest> > writeRequests = new Dictionary <string, List <WriteRequest> >();
writeRequests.Add(DataDefinitions.QUOTES_TABLE, new List <WriteRequest>());
foreach (Quote quote in batchQuotes)
{
PutRequest request = new PutRequest(QuoteExtensions.AttributesOfQuote(quote));
writeRequests[DataDefinitions.QUOTES_TABLE].Add(new WriteRequest(request));
}
BatchWriteItemRequest batchRequest = new BatchWriteItemRequest
{
RequestItems = writeRequests
};
Console.WriteLine("Before writing to dynamodb");
BatchWriteItemResponse batchResponse = await _client.BatchWriteItemAsync(batchRequest);
Console.WriteLine($"quotes batch response: {batchResponse.HttpStatusCode}");
}
/// <summary>
/// Delete multiple entries from a DynamoDB table (Batch delete)
/// </summary>
/// <param name="tableName">The name of the table to delete entries</param>
/// <param name="toDelete">List of key values for each entry that must be deleted in the batch</param>
/// <returns></returns>
public Task DeleteEntriesAsync(string tableName, IReadOnlyCollection <Dictionary <string, AttributeValue> > toDelete)
{
if (Logger.IsVerbose2)
{
Logger.Verbose2("Deleting {0} table entries", tableName);
}
if (toDelete == null)
{
throw new ArgumentNullException("collection");
}
if (toDelete.Count == 0)
{
return(TaskDone.Done);
}
try
{
var request = new BatchWriteItemRequest();
request.RequestItems = new Dictionary <string, List <WriteRequest> >();
var batch = new List <WriteRequest>();
foreach (var keys in toDelete)
{
var writeRequest = new WriteRequest();
writeRequest.DeleteRequest = new DeleteRequest();
writeRequest.DeleteRequest.Key = keys;
batch.Add(writeRequest);
}
request.RequestItems.Add(tableName, batch);
return(ddbClient.BatchWriteItemAsync(request));
}
catch (Exception exc)
{
Logger.Warn(ErrorCode.StorageProviderBase,
$"Intermediate error deleting entries from the table {tableName}.", exc);
throw;
}
}
private async Task WriteSeries(int?rowCount, bool runContinuous, bool useGroupedTable, string seriesKey)
{
using (var client = new AmazonDynamoDBClient(credentials, config))
{
var dataSet = new DataSet(seriesKey);
var writeToTableName = useGroupedTable
? groupedTableName
: tableName;
var records = useGroupedTable
? dataSet.GetRowGroups(rowCount, runContinuous).Select(CreateWriteRequest)
: dataSet.GetRows(rowCount, runContinuous).Select(CreateWriteRequest);
var count = 0;
foreach (var batch in records.Batch(25)) // DynamoDB supports writing 25 records at a time
{
var requestItems = new Dictionary <string, List <WriteRequest> >
{
[writeToTableName] = batch
};
while (requestItems.Any())
{
var request = new BatchWriteItemRequest()
{
RequestItems = requestItems
};
try
{
var response = await client.BatchWriteItemAsync(request);
// Attempt to write unprocessed items again
requestItems = response.UnprocessedItems;
}
catch (AmazonDynamoDBException ex)
{
await output.WriteLineAsync(ex.Message);
}
}
count += batch.Count;
Interlocked.Add(ref totalCount, batch.Count);
await output.WriteLineAsync($"Wrote {count} records from dataset '{dataSet.SeriesKey}' to: {writeToTableName}");
}
}
}
/// <summary>
/// Sends a list of write requests in batches as large as the AWS SDK will allow.
/// </summary>
/// <param name="client">the client</param>
/// <param name="tableName">the table name</param>
/// <param name="requests">list of requests</param>
/// <returns>async Task with no return value</returns>
public static async Task BatchWriteRequestsAsync(AmazonDynamoDBClient client, string tableName,
List <WriteRequest> requests)
{
int batchSize = 25;
for (int i = 0; i < requests.Count; i += batchSize)
{
var batch = requests.GetRange(i, Math.Min(batchSize, requests.Count - i));
var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> >()
{
{ tableName, batch }
});
await client.BatchWriteItemAsync(request);
}
}
private static async void CallBatchWriteTillCompletion(AmazonDynamoDBClient client, BatchWriteItemRequest request)
{
BatchWriteItemResponse response;
int callCount = 0;
do
{
Console.WriteLine("Making request");
response = await client.BatchWriteItemAsync(request);
callCount++;
// Check the response.
var tableConsumedCapacities = response.ConsumedCapacity;
var unprocessed = response.UnprocessedItems;
Console.WriteLine("Per-table consumed capacity");
foreach (var tableConsumedCapacity in tableConsumedCapacities)
{
Console.WriteLine("{0} - {1}", tableConsumedCapacity.TableName, tableConsumedCapacity.CapacityUnits);
}
Console.WriteLine("Unprocessed");
foreach (var unp in unprocessed)
{
Console.WriteLine("{0} - {1}", unp.Key, unp.Value.Count);
}
Console.WriteLine();
// For the next iteration, the request will have unprocessed items.
request.RequestItems = unprocessed;
} while (response.UnprocessedItems.Count > 0);
Console.WriteLine("Total # of batch write API calls made: {0}", callCount);
}
public async Task ClearAsync(CancellationToken cancellationToken)
{
Dictionary <string, AttributeValue> lastKeyEvaluated = null;
do
{
var request = new QueryRequest
{
TableName = _tableName,
KeyConditionExpression = "Dictionary = :v_Dictionary",
ExpressionAttributeValues = new Dictionary <string, AttributeValue>
{
{ ":v_Dictionary", new AttributeValue(DictionaryName) },
},
ExpressionAttributeNames = new Dictionary <string, string>()
{
{ "#K", "Key" } // MEMO: 'Key' is reserved keyword in DynamoDB Query.
},
ProjectionExpression = "Dictionary, #K",
};
var response = await _dynamoDBClient.QueryAsync(request, cancellationToken).ConfigureAwait(false);
if (cancellationToken.IsCancellationRequested)
{
return;
}
if (!response.Items.Any())
{
return;
}
var batchRequest = new BatchWriteItemRequest
{
RequestItems = new Dictionary <string, List <WriteRequest> >
{
{
_tableName,
response.Items.Select(x => new WriteRequest
{
DeleteRequest = new DeleteRequest
{
Key = new Dictionary <string, AttributeValue>
{
{ "Dictionary", x["Dictionary"] },
{ "Key", x["Key"] },
}
}
}).ToList()
}
}
};
await _dynamoDBClient.BatchWriteItemAsync(batchRequest, cancellationToken).ConfigureAwait(false);
if (cancellationToken.IsCancellationRequested)
{
return;
}
lastKeyEvaluated = response.LastEvaluatedKey;
} while (lastKeyEvaluated != null && lastKeyEvaluated.Count != 0);
}
/// <summary>
/// Example that writes to a DynamoDb more than one item in one call.
/// </summary>
/// <returns></returns>
public static async Task BatchWriteExampleAsync()
{
//Create Client
var client = new AmazonDynamoDBClient(RegionEndpoint.USWest2);
//Definition of the first item to put on a table
var putFirstItem = new PutRequest(new Dictionary <string, AttributeValue>
{
{ "pk", new AttributeValue("*****@*****.**") },
{ "sk", new AttributeValue("metadata") },
{ "attribute1", new AttributeValue("Attribute1 value") }
//Add other attributes as you need
}
);
//Definition of the second item to put on a table
var putSecondItem = new PutRequest(new Dictionary <string, AttributeValue>
{
{ "pk", new AttributeValue("*****@*****.**") },
{ "sk", new AttributeValue("metadata") },
{ "attribute1", new AttributeValue("Attribute1 value") }
//Add other attributes as you need
}
);
//Definition of an item to delete
var deleteItem = new DeleteRequest(new Dictionary <string, AttributeValue>
{
{ "pk", new AttributeValue("*****@*****.**") },
{ "sk", new AttributeValue("metadata") },
}
);
//Request that group all the previous Put & Delete actions
var writeRequest = new BatchWriteItemRequest
{
RequestItems = new Dictionary <string, List <WriteRequest> >
{
{
//Name of the table
"RetailDatabase",
new List <WriteRequest>()
{
new WriteRequest(putFirstItem),
new WriteRequest(putSecondItem),
new WriteRequest(deleteItem)
}
},//You can execute other collections of requests on other tables at the same time
{
//Name of the table
"RetailDatabase2",
new List <WriteRequest>()
{
new WriteRequest(putFirstItem),
new WriteRequest(putSecondItem),
new WriteRequest(deleteItem)
}
}
}
};
try
{
//Execution of the request
var responseWrite = await client.BatchWriteItemAsync(writeRequest);
Console.WriteLine($"Status {responseWrite.HttpStatusCode}");
Console.WriteLine($"Number of items not processed that you need to try again:{responseWrite.UnprocessedItems}");
}
catch (Exception e)
{
Console.Error.WriteLine(e.Message);
}
}
