static async Task SendToDynamoAsync(IEnumerable <object> entities, IAmazonDynamoDB client,
ICollection <Dictionary <string, List <WriteRequest> > > entitiesToRetry)
{
var item = new BatchWriteItemRequest
{
RequestItems = new Dictionary <string, List <WriteRequest> >
{
{
"mp-dyndb-entities-qa", entities.Select(t => new WriteRequest
{
PutRequest = new PutRequest(Document.FromJson(JsonConvert.SerializeObject(t))
.ToAttributeMap())
}).ToList()
}
}
};
var result = await client.BatchWriteItemAsync(item);
entitiesToRetry.Add(result.UnprocessedItems);
if (result.HttpStatusCode != HttpStatusCode.OK)
{
throw new Exception();
}
}
private static void CallBatchWriteTillCompletion(BatchWriteItemRequest request)
{
BatchWriteItemResponse response;
int callCount = 0;
do
{
Console.WriteLine("Making request");
response = client.BatchWriteItem(request);
callCount++;
// Check the response.
var tableConsumedCapacities = response.ConsumedCapacity;
var unprocessed = response.UnprocessedItems;
Console.WriteLine("Per-table consumed capacity");
foreach (var tableConsumedCapacity in tableConsumedCapacities)
{
Console.WriteLine("{0} - {1}", tableConsumedCapacity.TableName, tableConsumedCapacity.CapacityUnits);
}
Console.WriteLine("Unprocessed");
foreach (var unp in unprocessed)
{
Console.WriteLine("{0} - {1}", unp.Key, unp.Value.Count);
}
Console.WriteLine();
// For the next iteration, the request will have unprocessed items.
request.RequestItems = unprocessed;
} while (response.UnprocessedItems.Count > 0);
Console.WriteLine("Total # of batch write API calls made: {0}", callCount);
}
public async Task <PhotoModel> CreatePhoto(PhotoModel photo)
{
photo.Score = scoreCalculator.GetPhotoScore(photo);
logWriter.LogInformation($"{nameof(CreatePhoto)}({nameof(photo.Filename)} = '{photo.Filename}')");
try
{
var item = Mappers.PhotoModel.ToDbItem(photo);
var request = new PutItemRequest(tableName, item);
BatchWriteItemRequest hashtagsRequests = GetHashtagsRequests(photo.RawText, photo.Hashtags, photo, HashtagUpdateMode.CreateOnly);
await dynamoDbCore.PutItem(request);
if (hashtagsRequests != null)
{
await dynamoDbCore.BatchWriteItem(hashtagsRequests);
}
logWriter.LogInformation($"Photo record created for '{photo.Filename}'");
return(photo);
}
catch (Exception ex)
{
logWriter.LogError(ex, $"Error in {nameof(CreatePhoto)}(photo.Id = '{photo.PhotoId}'):\n{ex.ToString()}");
throw new Exception("Error when creating photo.", ex);
}
}
public async Task SetPhotoText(UserId userId, PhotoId photoId, string text, IEnumerable <HashtagModel> hashtags)
{
logWriter.LogInformation($"{nameof(SetPhotoText)}({nameof(userId)} = '{userId}', {nameof(photoId)} = '{photoId}', {nameof(text)} = '{text}')");
var existingItem = await GetPhotoById(photoId, userId);
BatchWriteItemRequest writeItemsRequest = GetHashtagsRequests(text, hashtags, existingItem, HashtagUpdateMode.Update);
var request = GetUpdateTextRequest(new PhotoModel
{
PhotoId = photoId,
UserId = userId,
RawText = text,
Hashtags = hashtags
});
logWriter.LogInformation($"UpdateItemRequest:\n{JsonConvert.SerializeObject(request)}");
logWriter.LogInformation($"BatchWriteItemRequest:\n{JsonConvert.SerializeObject(writeItemsRequest)}");
await dynamoDbCore.UpdateItem(request);
if (writeItemsRequest != null)
{
await dynamoDbCore.BatchWriteItem(writeItemsRequest);
}
}
private Task BatchInsertAsync(IEnumerable <Message> messages)
{
var requests = messages.Select(x =>
new WriteRequest(
new PutRequest(
new Dictionary <string, AttributeValue> {
[MESSAGE_ID] = new AttributeValue {
S = Guid.NewGuid().ToString()
},
[SOURCE] = new AttributeValue {
S = x.Source
},
[TEXT] = new AttributeValue {
S = x.Text
}
}
)
)
).ToList();
var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> > {
{ _tableName, requests }
});
return(_dynamoClient.BatchWriteItemAsync(request));
}
internal BatchWriteItemResponse BatchWriteItem(BatchWriteItemRequest request)
{
var marshaller = new BatchWriteItemRequestMarshaller();
var unmarshaller = BatchWriteItemResponseUnmarshaller.Instance;
return(Invoke <BatchWriteItemRequest, BatchWriteItemResponse>(request, marshaller, unmarshaller));
}
async Task PostAsync(IEnumerable <JObject> values, ILambdaContext?context)
{
context?.Logger.LogLine("Sending batch to dynamo");
var requests = values.Select(e => new WriteRequest
{
PutRequest = new PutRequest(Document.FromJson(e.ToString(Formatting.None)).ToAttributeMap())
});
var batch = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> >
{
[tableName] = requests.ToList()
});
var result = await dynamoClient.BatchWriteItemAsync(batch);
if (result.HttpStatusCode != HttpStatusCode.OK)
{
var metadata = JsonConvert.SerializeObject(result.ResponseMetadata.Metadata, new JsonSerializerSettings {
MaxDepth = 3
});
context?.Logger.LogLine($"Dynamo Call failed with {result.HttpStatusCode} {metadata}");
context?.Logger.LogLine($"Data: {string.Join(Environment.NewLine, values.Select(s => s.ToString()))}");
throw new Exception(metadata);
}
}
public static List <BatchWriteItemResponse> WriteItemsAsync(List <Dictionary <string, AttributeValue> > items)
{
var resultList = new List <BatchWriteItemResponse>();
for (int i = 0; i <= items.Count / 25; i++)
{
int index = i * 25;
int count = Math.Min(25, items.Count - index);
var batch = items.GetRange(index, count);
System.Diagnostics.Debug.WriteLine($"Processing Items {index}-{index + count}");
var request = new BatchWriteItemRequest();
var writes = new List <WriteRequest>();
foreach (var item in batch)
{
writes.Add(new WriteRequest(new PutRequest()
{
Item = item
}));
}
request.RequestItems.Add(Manager.DestinationTable.Name, writes);
var result = Manager.Client.BatchWriteItemAsync(request).Result;
resultList.Add(result);
}
return(resultList);
}
private static async Task <BatchWriteItemResponse> BuildDelete <Item>(Item[] items, string tableName, ItemConverter <Item> itemConverter, AmazonDynamoDBClient dynamodbClient)
{
List <WriteRequest> writeRequests = new List <WriteRequest>();
foreach (Item item in items)
{
DeleteRequest deleteRequest = new DeleteRequest()
{
Key = itemConverter.GetItemAsAttributes(item)
};
WriteRequest writeRequest = new WriteRequest()
{
DeleteRequest = deleteRequest
};
writeRequests.Add(writeRequest);
}
BatchWriteItemRequest batchWriteItemRequest = new BatchWriteItemRequest()
{
RequestItems = new Dictionary <string, List <WriteRequest> >()
{
{
tableName,
writeRequests
}
}
};
return(await dynamodbClient.BatchWriteItemAsync(batchWriteItemRequest));
}
// This processes 25 quotes at a time since the max is 25 as per dynamoDB batch write
private async Task WriteBatch(IEnumerable <Quote> batchQuotes)
{
// There is this possibility when there is nothing to approve
if (batchQuotes.Count() == 0)
{
Console.WriteLine("Batch empty");
return;
}
Console.WriteLine("Writing batch");
Dictionary <string, List <WriteRequest> > writeRequests = new Dictionary <string, List <WriteRequest> >();
writeRequests.Add(DataDefinitions.QUOTES_TABLE, new List <WriteRequest>());
foreach (Quote quote in batchQuotes)
{
PutRequest request = new PutRequest(QuoteExtensions.AttributesOfQuote(quote));
writeRequests[DataDefinitions.QUOTES_TABLE].Add(new WriteRequest(request));
}
BatchWriteItemRequest batchRequest = new BatchWriteItemRequest
{
RequestItems = writeRequests
};
Console.WriteLine("Before writing to dynamodb");
BatchWriteItemResponse batchResponse = await _client.BatchWriteItemAsync(batchRequest);
Console.WriteLine($"quotes batch response: {batchResponse.HttpStatusCode}");
}
private BatchWriteItemRequest ConstructRequest(Dictionary <string, QuickList <WriteRequestDocument> > writeItems, Table targetTable, out Dictionary <Key, Document> documentMap, bool isAsync)
{
documentMap = new Dictionary <Key, Document>(keyComparer);
BatchWriteItemRequest request = new BatchWriteItemRequest();
foreach (var writeItem in writeItems)
{
string tableName = writeItem.Key;
List <WriteRequestDocument> requestItems = writeItem.Value.GetItems();
if (requestItems != null && requestItems.Count > 0)
{
var table = tableMap[tableName];
var requestList = new List <WriteRequest>();
foreach (var item in requestItems)
{
requestList.Add(item.WriteRequest);
// Add document corresponding to the Put request to document map
if (item.WriteRequest.PutRequest != null)
{
var key = table.MakeKey(item.Document);
documentMap.Add(key, item.Document);
}
}
request.RequestItems[tableName] = requestList;
}
}
request.BeforeRequestEvent += isAsync ?
new RequestEventHandler(targetTable.UserAgentRequestEventHandlerAsync) :
new RequestEventHandler(targetTable.UserAgentRequestEventHandlerSync);
return(request);
}
public void PutItems <T>(IEnumerable <T> items)
{
var table = DynamoMetadata.GetTable <T>();
var remaining = items.ToList();
PopulateMissingHashes(table, remaining);
while (remaining.Count > 0)
{
var batchSize = Math.Min(remaining.Count, MaxWriteBatchSize);
var nextBatch = remaining.GetRange(0, batchSize);
remaining.RemoveRange(0, batchSize);
var putItems = nextBatch.Map(x => new WriteRequest(
new PutRequest(Converters.ToAttributeValues(this, x, table))));
var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> > {
{ table.Name, putItems }
});
var response = Exec(() => DynamoDb.BatchWriteItem(request));
var i = 0;
while (response.UnprocessedItems.Count > 0)
{
response = Exec(() => DynamoDb.BatchWriteItem(new BatchWriteItemRequest(response.UnprocessedItems)));
if (response.UnprocessedItems.Count > 0)
{
i.SleepBackOffMultiplier();
}
}
}
}
/// <summary>
/// <para>The <i>BatchWriteItem</i> operation puts or deletes multiple items in one or more tables. A single call to <i>BatchWriteItem</i> can
/// write up to 1 MB of data, which can comprise as many as 25 put or delete requests. Individual items to be written can be as large as 64
/// KB.</para> <para><b>NOTE:</b> BatchWriteItem cannot update items. To update items, use the UpdateItem API. </para> <para>The individual
/// <i>PutItem</i> and <i>DeleteItem</i> operations specified in <i>BatchWriteItem</i> are atomic; however <i>BatchWriteItem</i> as a whole is
/// not. If any requested operations fail because the table's provisioned throughput is exceeded or an internal processing failure occurs, the
/// failed operations are returned in the <i>UnprocessedItems</i> response parameter. You can investigate and optionally resend the requests.
/// Typically, you would call <i>BatchWriteItem</i> in a loop. Each iteration would check for unprocessed items and submit a new
/// <i>BatchWriteItem</i> request with those unprocessed items until all items have been processed.</para> <para>To write one item, you can use
/// the <i>PutItem</i> operation; to delete one item, you can use the <i>DeleteItem</i> operation.</para> <para>With <i>BatchWriteItem</i> , you
/// can efficiently write or delete large amounts of data, such as from Amazon Elastic MapReduce (EMR), or copy data from another database into
/// DynamoDB. In order to improve performance with these large-scale operations, <i>BatchWriteItem</i> does not behave in the same way as
/// individual <i>PutItem</i> and <i>DeleteItem</i> calls would For example, you cannot specify conditions on individual put and delete
/// requests, and <i>BatchWriteItem</i> does not return deleted items in the response.</para> <para>If you use a programming language that
/// supports concurrency, such as Java, you can use threads to write items in parallel. Your application must include the necessary logic to
/// manage the threads.</para> <para>With languages that don't support threading, such as PHP, <i>BatchWriteItem</i> will write or delete the
/// specified items one at a time. In both situations, <i>BatchWriteItem</i> provides an alternative where the API performs the specified put
/// and delete operations in parallel, giving you the power of the thread pool approach without having to introduce complexity into your
/// application.</para> <para>Parallel processing reduces latency, but each specified put and delete request consumes the same number of write
/// capacity units whether it is processed in parallel or not. Delete operations on nonexistent items consume one write capacity unit.</para>
/// <para>If one or more of the following is true, DynamoDB rejects the entire batch write operation:</para>
/// <ul>
/// <li> <para>One or more tables specified in the <i>BatchWriteItem</i> request does not exist.</para> </li>
/// <li> <para>Primary key attributes specified on an item in the request do not match those in the corresponding table's primary key
/// schema.</para> </li>
/// <li> <para>You try to perform multiple operations on the same item in the same <i>BatchWriteItem</i> request. For example, you cannot put
/// and delete the same item in the same <i>BatchWriteItem</i> request. </para> </li>
/// <li> <para>The total request size exceeds 1 MB.</para> </li>
/// <li> <para>Any individual item in a batch exceeds 64 KB.</para> </li>
///
/// </ul>
/// </summary>
///
/// <param name="batchWriteItemRequest">Container for the necessary parameters to execute the BatchWriteItem service method on
/// AmazonDynamoDBv2.</param>
///
/// <returns>The response from the BatchWriteItem service method, as returned by AmazonDynamoDBv2.</returns>
///
/// <exception cref="T:Amazon.DynamoDBv2.Model.ItemCollectionSizeLimitExceededException" />
/// <exception cref="T:Amazon.DynamoDBv2.Model.ResourceNotFoundException" />
/// <exception cref="T:Amazon.DynamoDBv2.Model.ProvisionedThroughputExceededException" />
/// <exception cref="T:Amazon.DynamoDBv2.Model.InternalServerErrorException" />
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
public Task <BatchWriteItemResponse> BatchWriteItemAsync(BatchWriteItemRequest batchWriteItemRequest, CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new BatchWriteItemRequestMarshaller();
var unmarshaller = BatchWriteItemResponseUnmarshaller.GetInstance();
return(Invoke <IRequest, BatchWriteItemRequest, BatchWriteItemResponse>(batchWriteItemRequest, marshaller, unmarshaller, signer, cancellationToken));
}
public async Task DeletePhoto(PhotoId photoId)
{
logWriter.LogInformation($"{nameof(DeletePhoto)}({nameof(photoId)} = '{photoId}'");
// Tirst we need to get the keys for all items to delete
IEnumerable <Dictionary <string, AttributeValue> > itemKeys = await GetAllItemKeys(photoId);
var request = new BatchWriteItemRequest
{
RequestItems = new Dictionary <string, List <WriteRequest> >
{
{
tableName,
itemKeys.Distinct(new DbKeyEqualityComparer()).Select(key => new WriteRequest {
DeleteRequest = new DeleteRequest
{
Key = key
}
}).ToList()
}
}
};
await dynamoDbCore.BatchWriteItem(request);
}
/// <summary>
/// Initiates the asynchronous execution of the BatchWriteItem operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the BatchWriteItem operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task <BatchWriteItemResponse> BatchWriteItemAsync(BatchWriteItemRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new BatchWriteItemRequestMarshaller();
var unmarshaller = BatchWriteItemResponseUnmarshaller.Instance;
return(InvokeAsync <BatchWriteItemRequest, BatchWriteItemResponse>(request, marshaller,
unmarshaller, cancellationToken));
}
private void CallUntilCompletion(BatchWriteItemRequest request, AmazonDynamoDB client)
{
do
{
var batchWriteItemResponse = client.BatchWriteItem(request);
var result = batchWriteItemResponse.BatchWriteItemResult;
request.RequestItems = result.UnprocessedItems;
} while (request.RequestItems.Count > 0);
}
public async Task DynamoDbUpdateCacheVideoDataAsync(List <WriteRequest> writeRequests)
{
var batchPutRequest = new BatchWriteItemRequest {
RequestItems = new Dictionary <string, List <WriteRequest> > {
{ _dynamoDbVideoTableName, writeRequests }
}
};
await _dynamoDbClient.BatchWriteItemAsync(batchPutRequest);
}
private static BatchWriteItemRequest GetNextBatch(BatchWriteItemRequest request, int skip)
{
var writeRequests = request.RequestItems.Skip(skip).Take(MaxNumberOfBatchWriteRequestItems).ToDictionary(x => x.Key, x => x.Value);
return(writeRequests.Any()
? new BatchWriteItemRequest {
RequestItems = writeRequests, ReturnConsumedCapacity = request.ReturnConsumedCapacity
}
: null);
}
private async Task CallUntilCompletionAsync(BatchWriteItemRequest request, Dictionary <string, Dictionary <Key, Document> > documentMap, IAmazonDynamoDB client, CancellationToken cancellationToken)
#endif
{
do
{
var result = await client.BatchWriteItemAsync(request, cancellationToken).ConfigureAwait(false);
request.RequestItems = result.UnprocessedItems;
Dictionary <Key, Document> unprocessedDocuments = new Dictionary <Key, Document>(keyComparer);
foreach (var unprocessedItems in result.UnprocessedItems)
{
string tableName = unprocessedItems.Key;
Table table = tableMap[tableName];
Dictionary <Key, Document> tableDocumentMap = documentMap[tableName];
foreach (var writeRequest in unprocessedItems.Value)
{
if (writeRequest.PutRequest != null)
{
var doc = table.FromAttributeMap(writeRequest.PutRequest.Item);
var key = table.MakeKey(doc);
Document document = null;
if (tableDocumentMap.TryGetValue(key, out document))
{
// Remove unprocessed requests from the document map
// and copy them to unprocessed documents.
unprocessedDocuments.Add(key, document);
tableDocumentMap.Remove(key);
}
}
}
// Commit the remaining documents in the document map
foreach (var document in tableDocumentMap.Values)
{
document.CommitChanges();
}
// Replace existing documents with just the unprocessed documents
documentMap[tableName] = unprocessedDocuments;
}
} while (request.RequestItems.Count > 0);
// Commit any remaining documents in document map.
// This would only happen if we are not able to match the items sent in the request
// with the items returned back as unprocessed items.
foreach (var tableDocumentMap in documentMap.Values)
{
foreach (var document in tableDocumentMap.Values)
{
document.CommitChanges();
}
}
}
private static int GetNumberOfWrites(BatchWriteItemRequest request)
{
int totalWrites = 0;
foreach (var writeItem in request.RequestItems)
{
var writes = writeItem.Value;
totalWrites += (writes == null ? 0 : writes.Count);
}
return(totalWrites);
}
public async Task SaveAsync(List <ScheduledTask> scheduledTasks)
{
var request = new BatchWriteItemRequest()
{
RequestItems = new Dictionary <string, List <WriteRequest> >()
{
{ Constants.ScheduledTasks, GetWriteItems(scheduledTasks) }
}
};
await _dbClient.BatchWriteItemAsync(request);
}
private BatchWriteItemRequest GetHashtagsRequests(string text, IEnumerable <HashtagModel> hashtags, PhotoModel existingItem, HashtagUpdateMode updateMode)
{
IEnumerable <HashtagModel> hashtagsBefore = updateMode == HashtagUpdateMode.Update
? existingItem.Hashtags
: emptyHashtagSequence;
existingItem.RawText = text;
existingItem.Hashtags = hashtags;
BatchWriteItemRequest writeItemsRequest = GetHashtagsWriteRequests(hashtags, existingItem, hashtagsBefore, updateMode);
return(writeItemsRequest);
}
/// <summary>
/// Removes items from a DynamoDB table.
/// </summary>
/// <param name="client">An initialized DynamoDB client object.</param>
/// <param name="tableName">The name of the table from which the item
/// will be deleted.</param>
/// <param name="idstring">The id of the item to be deleted.</param>
/// <param name="area">The area of the item to delete.</param>
/// <returns>The response from the call to BatchWriteItemAsync.</returns>
public static async Task <BatchWriteItemResponse> RemoveItemsAsync(
IAmazonDynamoDB client,
string tableName,
string idstring,
string area)
{
var ids = idstring.Split(" ");
var writeRequests = new List <WriteRequest>();
var items = new Dictionary <string, List <WriteRequest> >();
for (int i = 0; i < ids.Length; i++)
{
var writeRequest = new WriteRequest
{
// For the operation to delete an item, if you provide a primary key value
// that does not exist in the table, there is no error.
DeleteRequest = new DeleteRequest
{
Key = new Dictionary <string, AttributeValue>()
{
{
"ID",
new AttributeValue
{
S = ids[i],
}
},
{
"Area",
new AttributeValue
{
S = area,
}
},
},
},
};
writeRequests.Add(writeRequest);
}
items.Add(tableName, writeRequests);
var request = new BatchWriteItemRequest(items);
var response = await client.BatchWriteItemAsync(request);
return(response);
}
private async Task WriteSeries(int?rowCount, bool runContinuous, bool useGroupedTable, string seriesKey)
{
using (var client = new AmazonDynamoDBClient(credentials, config))
{
var dataSet = new DataSet(seriesKey);
var writeToTableName = useGroupedTable
? groupedTableName
: tableName;
var records = useGroupedTable
? dataSet.GetRowGroups(rowCount, runContinuous).Select(CreateWriteRequest)
: dataSet.GetRows(rowCount, runContinuous).Select(CreateWriteRequest);
var count = 0;
foreach (var batch in records.Batch(25)) // DynamoDB supports writing 25 records at a time
{
var requestItems = new Dictionary <string, List <WriteRequest> >
{
[writeToTableName] = batch
};
while (requestItems.Any())
{
var request = new BatchWriteItemRequest()
{
RequestItems = requestItems
};
try
{
var response = await client.BatchWriteItemAsync(request);
// Attempt to write unprocessed items again
requestItems = response.UnprocessedItems;
}
catch (AmazonDynamoDBException ex)
{
await output.WriteLineAsync(ex.Message);
}
}
count += batch.Count;
Interlocked.Add(ref totalCount, batch.Count);
await output.WriteLineAsync($"Wrote {count} records from dataset '{dataSet.SeriesKey}' to: {writeToTableName}");
}
}
}
internal BatchWriteItemResponse BatchWriteItem(BatchWriteItemRequest request)
{
var task = BatchWriteItemAsync(request);
try
{
return(task.Result);
}
catch (AggregateException e)
{
ExceptionDispatchInfo.Capture(e.InnerException).Throw();
return(null);
}
}
/// <summary>
/// Initiates the asynchronous execution of the BatchWriteItem operation.
/// <seealso cref="Amazon.DynamoDBv2.IAmazonDynamoDB"/>
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the BatchWriteItem operation.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
/// <returns>void</returns>
public void BatchWriteItemAsync(BatchWriteItemRequest request, AmazonServiceCallback callback, object state)
{
if (!AmazonInitializer.IsInitialized)
{
throw new Exception("AWSPrefab is not added to the scene");
}
ThreadPool.QueueUserWorkItem(new WaitCallback(delegate
{
var marshaller = new BatchWriteItemRequestMarshaller();
var unmarshaller = BatchWriteItemResponseUnmarshaller.Instance;
Invoke(request, callback, state, marshaller, unmarshaller, signer);
}));
return;
}
/// <summary>
/// Sends a list of write requests in batches as large as the AWS SDK will allow.
/// </summary>
/// <param name="client">the client</param>
/// <param name="tableName">the table name</param>
/// <param name="requests">list of requests</param>
/// <returns>async Task with no return value</returns>
public static async Task BatchWriteRequestsAsync(AmazonDynamoDBClient client, string tableName,
List <WriteRequest> requests)
{
int batchSize = 25;
for (int i = 0; i < requests.Count; i += batchSize)
{
var batch = requests.GetRange(i, Math.Min(batchSize, requests.Count - i));
var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> >()
{
{ tableName, batch }
});
await client.BatchWriteItemAsync(request);
}
}
private void CallUntilCompletion(BatchWriteItemRequest request, Dictionary <Key, Document> documentMap, AmazonDynamoDB client)
{
do
{
var batchWriteItemResponse = client.BatchWriteItem(request);
var result = batchWriteItemResponse.BatchWriteItemResult;
request.RequestItems = result.UnprocessedItems;
Dictionary <Key, Document> unprocessedDocuments = new Dictionary <Key, Document>(keyComparer);
foreach (var unprocessedItems in result.UnprocessedItems)
{
Table table = tableMap[unprocessedItems.Key];
foreach (var writeRequest in unprocessedItems.Value)
{
if (writeRequest.PutRequest != null)
{
var key = table.MakeKey(Document.FromAttributeMap(writeRequest.PutRequest.Item));
Document document = null;
if (documentMap.TryGetValue(key, out document))
{
// Remove unprocessed requests from the document map
// and copy them to unprocessed documents.
unprocessedDocuments.Add(key, document);
documentMap.Remove(key);
}
}
}
}
// Commit the remaining documents in the document map
foreach (var document in documentMap.Values)
{
document.CommitChanges();
}
documentMap = unprocessedDocuments;
} while (request.RequestItems.Count > 0);
// Commit any remaining documents in document map.
// This would only happen if we are not able to match the items sent in the request
// with the items returned back as unprocessed items.
foreach (var document in documentMap.Values)
{
document.CommitChanges();
}
}
private List <string> WriteBigBatch(string hashTableName, int items, int itemsStartingIndex, int itemSize)
{
List <string> itemIds = new List <string>();
string itemData = new string('@', itemSize);
List <WriteRequest> writeRequests = new List <WriteRequest>();
for (int i = 0; i < items; i++)
{
var itemId = (itemsStartingIndex + i).ToString();
itemIds.Add(itemId);
var writeRequest = new WriteRequest
{
PutRequest = new PutRequest
{
Item = new Dictionary <string, AttributeValue>
{
{ "Id", new AttributeValue {
N = itemId
} },
{ "Data", new AttributeValue {
S = itemData
} }
}
}
};
writeRequests.Add(writeRequest);
}
var request = new BatchWriteItemRequest
{
RequestItems = new Dictionary <string, List <WriteRequest> >
{
{ hashTableName, writeRequests }
}
};
BatchWriteItemResult result;
do
{
result = Client.BatchWriteItem(request);
request.RequestItems = result.UnprocessedItems;
} while (result.UnprocessedItems != null && result.UnprocessedItems.Count > 0);
return(itemIds);
}
private Task BatchDeleteAsync(IEnumerable <string> messageIds)
{
var requests = messageIds.Select(x =>
new WriteRequest(
new DeleteRequest(
new Dictionary <string, AttributeValue> {
{ MESSAGE_ID, new AttributeValue(x) }
}
)
)
).ToList();
var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> > {
{ _tableName, requests }
});
return(_dynamoClient.BatchWriteItemAsync(request));
}
