/// <summary> /// Specify the BatchGetItems operation for a list of primary keys that all share the same record type. /// </summary> /// <param name="primaryKeys">List of primary keys to retrieve.</param> /// <param name="consistentRead">Boolean indicating if the read operations should be performed against the main partition (2x cost compared to eventual consistent read).</param> /// <typeparam name="TRecord">The record type.</typeparam> public IDynamoTableBatchGetItems <TRecord> BatchGetItems <TRecord>(IEnumerable <DynamoPrimaryKey <TRecord> > primaryKeys, bool consistentRead) where TRecord : class { if (!primaryKeys.Any()) { throw new ArgumentException("primary keys cannot be empty", nameof(primaryKeys)); } if (primaryKeys.Count() > 100) { throw new ArgumentException("too many primary keys", nameof(primaryKeys)); } var request = new BatchGetItemRequest { RequestItems = { [TableName] = new KeysAndAttributes { ConsistentRead = consistentRead, Keys = primaryKeys.Select(primaryKey => new Dictionary <string, AttributeValue> { [primaryKey.PKName] = new AttributeValue(primaryKey.PKValue), [primaryKey.SKName] = new AttributeValue(primaryKey.SKValue) }).ToList() } } }; return(new DynamoTableBatchGetItems <TRecord>(this, request)); }
private Results GetAttributeItems() { var results = new Results(Batches); if (Batches == null || Batches.Count == 0) { return(results); } // use client from the table from the first batch var firstBatch = this.Batches[0]; var targetTable = firstBatch.TargetTable; var clientToUse = targetTable.DDBClient; var convertedBatches = ConvertBatches(); while (true) { var nextSet = GetNextRequestItems(ref convertedBatches, MaxItemsPerCall); if (nextSet.Count == 0) { break; } BatchGetItemRequest request = CreateRequest(nextSet); targetTable.AddRequestHandler(request, isAsync: false); CallUntilCompletion(clientToUse, request, results); } return(results); }
private List <T> ConvertBatchGetItemResponse <T>(DynamoMetadataType table, KeysAndAttributes getItems) { var to = new List <T>(); var request = new BatchGetItemRequest(new Dictionary <string, KeysAndAttributes> { { table.Name, getItems } }); var response = Exec(() => DynamoDb.BatchGetItem(request)); List <Dictionary <string, AttributeValue> > results; if (response.Responses.TryGetValue(table.Name, out results)) { results.Each(x => to.Add(Converters.FromAttributeValues <T>(table, x))); } var i = 0; while (response.UnprocessedKeys.Count > 0) { response = Exec(() => DynamoDb.BatchGetItem(new BatchGetItemRequest(response.UnprocessedKeys))); if (response.Responses.TryGetValue(table.Name, out results)) { results.Each(x => to.Add(Converters.FromAttributeValues <T>(table, x))); } if (response.UnprocessedKeys.Count > 0) { i.SleepBackOffMultiplier(); } } return(to); }
private async Task <List <Quote> > getQuotesByIds(IEnumerable <string> ids) { // ETL List <Dictionary <string, AttributeValue> > listOfIdsToBeFetched = new List <Dictionary <string, AttributeValue> >(); foreach (string id in ids) { listOfIdsToBeFetched.Add(new Dictionary <string, AttributeValue> { { DataDefinitions.QUOTES_TABLE_HASH_KEY, new AttributeValue { N = id } } }); } BatchGetItemRequest request = new BatchGetItemRequest(new Dictionary <string, KeysAndAttributes> { { DataDefinitions.QUOTES_TABLE, new KeysAndAttributes { Keys = listOfIdsToBeFetched } } }); BatchGetItemResponse response = await _client.BatchGetItemAsync(request); // Extract data from the complex structure of response // Will return records List <Quote> quotes = new List <Quote>(); foreach (Dictionary <string, AttributeValue> pair in response.Responses[DataDefinitions.QUOTES_TABLE]) { quotes.Add(Quote.ToQuoteFromTable(pair)); } return(quotes); }
private static BatchGetItemRequest CreateRequest(Dictionary <string, RequestSet> set, Table targetTable, bool isAsync) { BatchGetItemRequest request = new BatchGetItemRequest(); ((Amazon.Runtime.Internal.IAmazonWebServiceRequest)request).AddBeforeRequestHandler(isAsync ? new RequestEventHandler(targetTable.UserAgentRequestEventHandlerAsync) : new RequestEventHandler(targetTable.UserAgentRequestEventHandlerSync) ); var requestItems = new Dictionary <string, KeysAndAttributes>(); foreach (var kvp in set) { var tableName = kvp.Key; var requestSet = kvp.Value; var keys = new KeysAndAttributes { Keys = requestSet.GetItems(), ConsistentRead = requestSet.Batch.ConsistentRead, AttributesToGet = requestSet.Batch.AttributesToGet }; requestItems.Add(tableName, keys); } request.RequestItems = requestItems; return(request); }
/// <summary> /// <para>The <i>BatchGetItem</i> operation returns the attributes of one or more items from one or more tables. You identify requested items by /// primary key.</para> <para>A single operation can retrieve up to 1 MB of data, which can contain as many as 100 items. <i>BatchGetItem</i> /// will return a partial result if the response size limit is exceeded, the table's provisioned throughput is exceeded, or an internal /// processing failure occurs. If a partial result is returned, the operation returns a value for <i>UnprocessedKeys</i> . You can use this /// value to retry the operation starting with the next item to get.</para> <para>For example, if you ask to retrieve 100 items, but each /// individual item is 50 KB in size, the system returns 20 items (1 MB) and an appropriate <i>UnprocessedKeys</i> value so you can get the next /// page of results. If desired, your application can include its own logic to assemble the pages of results into one dataset.</para> <para>If /// no items can be processed because of insufficient provisioned throughput on each of the tables involved in the request, <i>BatchGetItem</i> /// throws <i>ProvisionedThroughputExceededException</i> . </para> <para>By default, <i>BatchGetItem</i> performs eventually consistent reads on /// every table in the request. If you want strongly consistent reads instead, you can set <i>ConsistentRead</i> to <c>true</c> for any or all /// tables.</para> <para>In order to minimize response latency, <i>BatchGetItem</i> retrieves items in parallel.</para> <para>When designing /// your application, keep in mind that DynamoDB does not return attributes in any particular order. To help parse the response by item, include /// the primary key values for the items in your request in the <i>AttributesToGet</i> parameter.</para> <para>If a requested item does not /// exist, it is not returned in the result. Requests for nonexistent items consume the minimum read capacity units according to the type of /// read. For more information, see <a href="http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/WorkingWithDDTables.html#CapacityUnitCalculations">Capacity Units /// Calculations</a> in the Amazon DynamoDB Developer Guide.</para> /// </summary> /// /// <param name="batchGetItemRequest">Container for the necessary parameters to execute the BatchGetItem service method on /// AmazonDynamoDBv2.</param> /// /// <returns>The response from the BatchGetItem service method, as returned by AmazonDynamoDBv2.</returns> /// /// <exception cref="T:Amazon.DynamoDBv2.Model.ResourceNotFoundException" /> /// <exception cref="T:Amazon.DynamoDBv2.Model.ProvisionedThroughputExceededException" /> /// <exception cref="T:Amazon.DynamoDBv2.Model.InternalServerErrorException" /> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> public Task <BatchGetItemResponse> BatchGetItemAsync(BatchGetItemRequest batchGetItemRequest, CancellationToken cancellationToken = default(CancellationToken)) { var marshaller = new BatchGetItemRequestMarshaller(); var unmarshaller = BatchGetItemResponseUnmarshaller.GetInstance(); return(Invoke <IRequest, BatchGetItemRequest, BatchGetItemResponse>(batchGetItemRequest, marshaller, unmarshaller, signer, cancellationToken)); }
internal BatchGetItemResponse BatchGetItem(BatchGetItemRequest request) { var marshaller = new BatchGetItemRequestMarshaller(); var unmarshaller = BatchGetItemResponseUnmarshaller.Instance; return(Invoke <BatchGetItemRequest, BatchGetItemResponse>(request, marshaller, unmarshaller)); }
private void GetAttributeItems(bool isAsync) { if (Batches == null || Batches.Count == 0) { return; } var firstBatch = this.Batches[0]; var targetTable = firstBatch.TargetTable; var client = targetTable.DDBClient; var convertedBatches = ConvertBatches(); while (true) { var nextSet = GetNextRequestItems(ref convertedBatches, MaxItemsPerCall); if (nextSet.Count == 0) { break; } BatchGetItemRequest request = CreateRequest(nextSet, targetTable, isAsync); CallUntilCompletion(client, request); } }
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems) { AmazonDynamoDBConfig config = new AmazonDynamoDBConfig(); config.RegionEndpoint = region; ConfigureClient(config); AmazonDynamoDBClient client = new AmazonDynamoDBClient(creds, config); BatchGetItemResponse resp = new BatchGetItemResponse(); do { BatchGetItemRequest req = new BatchGetItemRequest { RequestItems = resp.UnprocessedKeys }; resp = client.BatchGetItem(req); CheckError(resp.HttpStatusCode, "200"); foreach (var obj in resp.Responses) { AddObject(obj); } foreach (var obj in resp.UnprocessedKeys) { AddObject(obj); } }while (resp.UnprocessedKeys.Count > 0); }
public async Task <IList <NovelGramUser> > GetUserBatchAsync(IList <string> userIds) { await this.dynamoBuilder.BuildUserTable(); if (userIds == null || !userIds.Any()) { return(null); } var batchRequest = new BatchGetItemRequest { RequestItems = new Dictionary <string, KeysAndAttributes> { { this.userTableName, new KeysAndAttributes { Keys = new List <Dictionary <string, AttributeValue> >(userIds.Select(GetKey)) } } } }; var response = await this.dynamoClient.BatchGetItemAsync(batchRequest); return(response?.Responses?.FirstOrDefault().Value?.Select(UserFromItem).ToArray()); }
public async Task <List <BatchGetItemResponse> > DynamoDbGetCacheVideoDataAsync(List <string> videoIds) { var listOfResponses = new List <BatchGetItemResponse>(); var batchVideoRequests = new List <List <Dictionary <string, AttributeValue> > >(); var batchVideoIds = videoIds.Select(x => new Dictionary <string, AttributeValue> { { "videoId", new AttributeValue { S = x } } }).ToList(); for (int i = 0; i < batchVideoIds.Count; i += 25) { batchVideoRequests.Add(batchVideoIds.GetRange(i, Math.Min(25, batchVideoIds.Count - i))); } foreach (var batchVideo in batchVideoRequests) { var getBatchRequest = new BatchGetItemRequest { RequestItems = new Dictionary <string, KeysAndAttributes> { { _dynamoDbVideoTableName, new KeysAndAttributes { Keys = batchVideo } } } }; listOfResponses.Add(await _dynamoDbClient.BatchGetItemAsync(getBatchRequest)); } return(listOfResponses); }
/// <summary> /// Initiates the asynchronous execution of the BatchGetItem operation. /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the BatchGetItem operation.</param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// <returns>The task object representing the asynchronous operation.</returns> public Task <BatchGetItemResponse> BatchGetItemAsync(BatchGetItemRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken)) { var marshaller = new BatchGetItemRequestMarshaller(); var unmarshaller = BatchGetItemResponseUnmarshaller.Instance; return(InvokeAsync <BatchGetItemRequest, BatchGetItemResponse>(request, marshaller, unmarshaller, cancellationToken)); }
public void BatchGetItemRequestTest1() { var request = new BatchGetItemRequest( new TableKeys("Table1", new RecordKey("id", 1), new RecordKey("id", 2)), new TableKeys("Table2", new RecordKey("id", 3), new RecordKey("id", 4)) ); var expected = @"{""RequestItems"":{""Table1"":{""Keys"":[{""id"":{""N"":""1""}},{""id"":{""N"":""2""}}]},""Table2"":{""Keys"":[{""id"":{""N"":""3""}},{""id"":{""N"":""4""}}]}}}"; Assert.Equal(expected, request.ToJson().ToString(pretty: false)); }
public async Task <IEnumerable <WorkflowInstance> > GetWorkflowInstances(IEnumerable <string> ids) { if (ids == null) { return(new List <WorkflowInstance>()); } var keys = new KeysAndAttributes() { Keys = new List <Dictionary <string, AttributeValue> >() }; foreach (var id in ids) { var key = new Dictionary <string, AttributeValue>() { { "id", new AttributeValue { S = id } } }; keys.Keys.Add(key); } var request = new BatchGetItemRequest { RequestItems = new Dictionary <string, KeysAndAttributes>() { { $"{_tablePrefix}-{WORKFLOW_TABLE}", keys } } }; var result = new List <Dictionary <string, AttributeValue> >(); BatchGetItemResponse response; do { response = await _client.BatchGetItemAsync(request); foreach (var tableResponse in response.Responses) { result.AddRange(tableResponse.Value); } request.RequestItems = response.UnprocessedKeys; } while (response.UnprocessedKeys.Count > 0); return(result.Select(i => i.ToWorkflowInstance())); }
internal BatchGetItemResponse BatchGetItem(BatchGetItemRequest request) { var task = BatchGetItemAsync(request); try { return(task.Result); } catch (AggregateException e) { ExceptionDispatchInfo.Capture(e.InnerException).Throw(); return(null); } }
/// <summary> /// Initiates the asynchronous execution of the BatchGetItem operation. /// <seealso cref="Amazon.DynamoDBv2.IAmazonDynamoDB"/> /// </summary> /// <param name="request">Container for the necessary parameters to execute the BatchGetItem operation.</param> /// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes</param> /// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property.</param> /// <returns>void</returns> public void BatchGetItemAsync(BatchGetItemRequest request, AmazonServiceCallback callback, object state) { if (!AmazonInitializer.IsInitialized) { throw new Exception("AWSPrefab is not added to the scene"); } ThreadPool.QueueUserWorkItem(new WaitCallback(delegate { var marshaller = new BatchGetItemRequestMarshaller(); var unmarshaller = BatchGetItemResponseUnmarshaller.Instance; Invoke(request, callback, state, marshaller, unmarshaller, signer); })); return; }
private static void CallUntilCompletion(IAmazonDynamoDB client, BatchGetItemRequest request, Results allResults) #endif { do { var serviceResponse = client.BatchGetItem(request); foreach (var kvp in serviceResponse.Responses) { var tableName = kvp.Key; var items = kvp.Value; allResults.Add(tableName, items); } request.RequestItems = serviceResponse.UnprocessedKeys; } while (request.RequestItems.Count > 0); }
private static async Task CallUntilCompletionAsync(IAmazonDynamoDB client, BatchGetItemRequest request, Results allResults, CancellationToken cancellationToken) #endif { do { var serviceResponse = await client.BatchGetItemAsync(request, cancellationToken).ConfigureAwait(false); foreach (var kvp in serviceResponse.Responses) { var tableName = kvp.Key; var items = kvp.Value; allResults.Add(tableName, items); } request.RequestItems = serviceResponse.UnprocessedKeys; } while (request.RequestItems.Count > 0); }
async Task <BatchGetItemResponse> IAmazonDynamoDB.BatchGetItemAsync(BatchGetItemRequest request, CancellationToken cancellationToken) { try { var response = await _dynamoDBClient.BatchGetItemAsync(request, cancellationToken); _callback?.Invoke(new { Action = nameof(IAmazonDynamoDB.BatchGetItemAsync), Request = request, Response = response }); return(response); } catch (Exception e) { _callback?.Invoke(new { Action = nameof(IAmazonDynamoDB.BatchGetItemAsync), Request = request, Exception = new { Type = e.GetType().FullName, Message = e.Message } }); throw; } }
private int GetBigBatch(string hashTableName, List <string> idsToGet) { var keys = new List <Dictionary <string, AttributeValue> >(); foreach (var id in idsToGet) { keys.Add(new Dictionary <string, AttributeValue> { { "Id", new AttributeValue { N = id } } }); } var request = new BatchGetItemRequest { RequestItems = new Dictionary <string, KeysAndAttributes> { { hashTableName, new KeysAndAttributes { Keys = keys } } } }; BatchGetItemResult result; int itemsRetrieved = 0; do { result = Client.BatchGetItem(request); itemsRetrieved += result.Responses[hashTableName].Count; request.RequestItems = result.UnprocessedKeys; } while (result.UnprocessedKeys != null && result.UnprocessedKeys.Count > 0); return(itemsRetrieved); }
private static BatchGetItemRequest CreateRequest(Dictionary <string, RequestSet> set) { BatchGetItemRequest request = new BatchGetItemRequest(); var requestItems = new Dictionary <string, KeysAndAttributes>(); foreach (var kvp in set) { var tableName = kvp.Key; var requestSet = kvp.Value; var keys = new KeysAndAttributes { Keys = requestSet.GetItems(), ConsistentRead = requestSet.Batch.ConsistentRead, AttributesToGet = requestSet.Batch.AttributesToGet }; requestItems.Add(tableName, keys); } request.RequestItems = requestItems; return(request); }
private void CallUntilCompletion(IAmazonDynamoDB client, BatchGetItemRequest request) #endif { do { var result = client.BatchGetItem(request); var responses = result.Responses; foreach (var response in responses) { var tableName = response.Key; var items = response.Value; List <Dictionary <string, AttributeValue> > fetchedItems; if (!AllRetrievedItems.TryGetValue(tableName, out fetchedItems)) { fetchedItems = new List <Dictionary <string, AttributeValue> >(); AllRetrievedItems[tableName] = fetchedItems; } fetchedItems.AddRange(items); } request.RequestItems = result.UnprocessedKeys; } while (request.RequestItems.Count > 0); }
public BatchGetItemHttpContent(BatchGetItemRequest request, string?tableNamePrefix) : base("DynamoDB_20120810.BatchGetItem") { _request = request; _tableNamePrefix = tableNamePrefix; }
private static async Task CallUntilCompletionAsync(AmazonDynamoDBClient client, BatchGetItemRequest request, Results allResults, CancellationToken cancellationToken)
private static void RetrieveMultipleItemsBatchGet() { var request = new BatchGetItemRequest { RequestItems = new Dictionary <string, KeysAndAttributes>() { { table1Name, new KeysAndAttributes { Keys = new List <Dictionary <string, AttributeValue> >() { new Dictionary <string, AttributeValue>() { { "Name", new AttributeValue { S = "Amazon DynamoDB" } } }, new Dictionary <string, AttributeValue>() { { "Name", new AttributeValue { S = "Amazon S3" } } } } } }, { table2Name, new KeysAndAttributes { Keys = new List <Dictionary <string, AttributeValue> >() { new Dictionary <string, AttributeValue>() { { "ForumName", new AttributeValue { S = "Amazon DynamoDB" } }, { "Subject", new AttributeValue { S = "DynamoDB Thread 1" } } }, new Dictionary <string, AttributeValue>() { { "ForumName", new AttributeValue { S = "Amazon DynamoDB" } }, { "Subject", new AttributeValue { S = "DynamoDB Thread 2" } } }, new Dictionary <string, AttributeValue>() { { "ForumName", new AttributeValue { S = "Amazon S3" } }, { "Subject", new AttributeValue { S = "S3 Thread 1" } } } } } } } }; BatchGetItemResponse response; do { Console.WriteLine("Making request"); response = client.BatchGetItem(request); // Check the response. var responses = response.Responses; // Attribute list in the response. foreach (var tableResponse in responses) { var tableResults = tableResponse.Value; Console.WriteLine("Items retrieved from table {0}", tableResponse.Key); foreach (var item1 in tableResults) { PrintItem(item1); } } // Any unprocessed keys? could happen if you exceed ProvisionedThroughput or some other error. Dictionary <string, KeysAndAttributes> unprocessedKeys = response.UnprocessedKeys; foreach (var unprocessedTableKeys in unprocessedKeys) { // Print table name. Console.WriteLine(unprocessedTableKeys.Key); // Print unprocessed primary keys. foreach (var key in unprocessedTableKeys.Value.Keys) { PrintItem(key); } } request.RequestItems = unprocessedKeys; } while (response.UnprocessedKeys.Count > 0); }
private static void CallUntilCompletion(AmazonDynamoDBClient client, BatchGetItemRequest request, Results allResults)
private void CallUntilCompletion(AmazonDynamoDBClient client, BatchGetItemRequest request)
public static BatchGetItemResponse BatchGetItem(this IAmazonDynamoDB client, BatchGetItemRequest request) { return(client.BatchGetItemAsync(request).GetResult()); }
private Dictionary <string, List <Dictionary <string, AttributeValue> > > GetAttributeItems(bool isAsync) { var allItems = new Dictionary <string, List <Dictionary <string, AttributeValue> > >(); if (Batches == null || Batches.Count == 0) { return(allItems); } DocumentBatchGet firstBatch = this.Batches[0]; BatchGetItemRequest request = new BatchGetItemRequest(); request.BeforeRequestEvent += isAsync ? new RequestEventHandler(firstBatch.TargetTable.UserAgentRequestEventHandlerAsync) : new RequestEventHandler(firstBatch.TargetTable.UserAgentRequestEventHandlerSync); foreach (var batch in Batches) { if (batch.Keys != null && batch.Keys.Count > 0) { if (request.RequestItems.ContainsKey(batch.TargetTable.TableName)) { throw new InvalidOperationException("Multiple batches refer to the same table."); } request.RequestItems.Add( batch.TargetTable.TableName, new KeysAndAttributes { Keys = batch.Keys, AttributesToGet = batch.AttributesToGet, ConsistentRead = batch.ConsistentRead }); } } var client = firstBatch.TargetTable.DDBClient; do { var batchGetItemResponse = client.BatchGetItem(request); var result = batchGetItemResponse.BatchGetItemResult; var responses = result.Responses; foreach (var response in responses) { string tableName = response.Key; BatchResponse batchResponse = response.Value; List <Dictionary <string, AttributeValue> > items = batchResponse.Items; List <Dictionary <string, AttributeValue> > fetchedItems; if (!allItems.TryGetValue(tableName, out fetchedItems)) { fetchedItems = new List <Dictionary <string, AttributeValue> >(); allItems[tableName] = fetchedItems; } fetchedItems.AddRange(items); } request.RequestItems = result.UnprocessedKeys; } while (request.RequestItems.Count > 0); return(allItems); }
public async Task <PhotoModel> GetPhotoById(PhotoId photoId, UserId currentUserId) { PhotoModel result = null; bool photoIsLikedByCurrentUser = false; logWriter.LogInformation($"{nameof(GetPhotoById)}({nameof(photoId)} = '{photoId}')"); var request = new BatchGetItemRequest { RequestItems = new Dictionary <string, KeysAndAttributes> { { tableName, new KeysAndAttributes { Keys = new List <Dictionary <string, AttributeValue> > { Mappers.PhotoModel.ToDbKey(new PhotoModel { PhotoId = photoId }), Mappers.PhotoLike.ToDbKey(new PhotoLikeRecord { PhotoId = photoId, UserId = currentUserId }) } } } } }; logWriter.LogInformation($"Batch get keys:\n{JsonConvert.SerializeObject(request)}"); try { var getItemResponse = await dynamoDbCore.BatchGetItem(request); foreach (var item in getItemResponse.SelectMany(r => r.Value)) { logWriter.LogInformation($"Record type: {item[FieldMappings.RecordType].S}"); switch (item[FieldMappings.RecordType].S.ToLowerInvariant()) { case "photo": result = Mappers.PhotoModel.FromDbItem(item); break; case "photolike": photoIsLikedByCurrentUser = true; break; } } if (result != null) { result.PhotoIsLikedByCurrentUser = photoIsLikedByCurrentUser; } return(result); } catch (Exception ex) { logWriter.LogError(ex, $"{nameof(GetPhotoById)}({nameof(photoId)} = '{photoId}'):\n{ex.ToString()}"); throw; } }