Пример #1
0
        static async Task SendToDynamoAsync(IEnumerable <object> entities, IAmazonDynamoDB client,
                                            ICollection <Dictionary <string, List <WriteRequest> > > entitiesToRetry)
        {
            var item = new BatchWriteItemRequest
            {
                RequestItems = new Dictionary <string, List <WriteRequest> >
                {
                    {
                        "mp-dyndb-entities-qa", entities.Select(t => new WriteRequest
                        {
                            PutRequest = new PutRequest(Document.FromJson(JsonConvert.SerializeObject(t))
                                                        .ToAttributeMap())
                        }).ToList()
                    }
                }
            };
            var result = await client.BatchWriteItemAsync(item);

            entitiesToRetry.Add(result.UnprocessedItems);

            if (result.HttpStatusCode != HttpStatusCode.OK)
            {
                throw new Exception();
            }
        }
Пример #2
0
        private static void CallBatchWriteTillCompletion(BatchWriteItemRequest request)
        {
            BatchWriteItemResponse response;

            int callCount = 0;
            do
            {
                Console.WriteLine("Making request");
                response = client.BatchWriteItem(request);
                callCount++;

                // Check the response.

                var tableConsumedCapacities = response.ConsumedCapacity;
                var unprocessed = response.UnprocessedItems;

                Console.WriteLine("Per-table consumed capacity");
                foreach (var tableConsumedCapacity in tableConsumedCapacities)
                {
                    Console.WriteLine("{0} - {1}", tableConsumedCapacity.TableName, tableConsumedCapacity.CapacityUnits);
                }

                Console.WriteLine("Unprocessed");
                foreach (var unp in unprocessed)
                {
                    Console.WriteLine("{0} - {1}", unp.Key, unp.Value.Count);
                }
                Console.WriteLine();

                // For the next iteration, the request will have unprocessed items.
                request.RequestItems = unprocessed;
            } while (response.UnprocessedItems.Count > 0);

            Console.WriteLine("Total # of batch write API calls made: {0}", callCount);
        }
        public async Task <PhotoModel> CreatePhoto(PhotoModel photo)
        {
            photo.Score = scoreCalculator.GetPhotoScore(photo);

            logWriter.LogInformation($"{nameof(CreatePhoto)}({nameof(photo.Filename)} = '{photo.Filename}')");
            try
            {
                var item    = Mappers.PhotoModel.ToDbItem(photo);
                var request = new PutItemRequest(tableName, item);
                BatchWriteItemRequest hashtagsRequests = GetHashtagsRequests(photo.RawText, photo.Hashtags, photo, HashtagUpdateMode.CreateOnly);

                await dynamoDbCore.PutItem(request);

                if (hashtagsRequests != null)
                {
                    await dynamoDbCore.BatchWriteItem(hashtagsRequests);
                }

                logWriter.LogInformation($"Photo record created for '{photo.Filename}'");
                return(photo);
            }
            catch (Exception ex)
            {
                logWriter.LogError(ex, $"Error in {nameof(CreatePhoto)}(photo.Id = '{photo.PhotoId}'):\n{ex.ToString()}");
                throw new Exception("Error when creating photo.", ex);
            }
        }
        public async Task SetPhotoText(UserId userId, PhotoId photoId, string text, IEnumerable <HashtagModel> hashtags)
        {
            logWriter.LogInformation($"{nameof(SetPhotoText)}({nameof(userId)} = '{userId}', {nameof(photoId)} = '{photoId}', {nameof(text)} = '{text}')");

            var existingItem = await GetPhotoById(photoId, userId);

            BatchWriteItemRequest writeItemsRequest = GetHashtagsRequests(text, hashtags, existingItem, HashtagUpdateMode.Update);

            var request = GetUpdateTextRequest(new PhotoModel
            {
                PhotoId  = photoId,
                UserId   = userId,
                RawText  = text,
                Hashtags = hashtags
            });

            logWriter.LogInformation($"UpdateItemRequest:\n{JsonConvert.SerializeObject(request)}");
            logWriter.LogInformation($"BatchWriteItemRequest:\n{JsonConvert.SerializeObject(writeItemsRequest)}");

            await dynamoDbCore.UpdateItem(request);

            if (writeItemsRequest != null)
            {
                await dynamoDbCore.BatchWriteItem(writeItemsRequest);
            }
        }
        private Task BatchInsertAsync(IEnumerable <Message> messages)
        {
            var requests = messages.Select(x =>
                                           new WriteRequest(
                                               new PutRequest(
                                                   new Dictionary <string, AttributeValue> {
                [MESSAGE_ID] = new AttributeValue {
                    S = Guid.NewGuid().ToString()
                },
                [SOURCE] = new AttributeValue {
                    S = x.Source
                },
                [TEXT] = new AttributeValue {
                    S = x.Text
                }
            }
                                                   )
                                               )
                                           ).ToList();
            var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> > {
                { _tableName, requests }
            });

            return(_dynamoClient.BatchWriteItemAsync(request));
        }
        internal BatchWriteItemResponse BatchWriteItem(BatchWriteItemRequest request)
        {
            var marshaller   = new BatchWriteItemRequestMarshaller();
            var unmarshaller = BatchWriteItemResponseUnmarshaller.Instance;

            return(Invoke <BatchWriteItemRequest, BatchWriteItemResponse>(request, marshaller, unmarshaller));
        }
Пример #7
0
        async Task PostAsync(IEnumerable <JObject> values, ILambdaContext?context)
        {
            context?.Logger.LogLine("Sending batch to dynamo");

            var requests = values.Select(e => new WriteRequest
            {
                PutRequest = new PutRequest(Document.FromJson(e.ToString(Formatting.None)).ToAttributeMap())
            });

            var batch = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> >
            {
                [tableName] = requests.ToList()
            });

            var result = await dynamoClient.BatchWriteItemAsync(batch);

            if (result.HttpStatusCode != HttpStatusCode.OK)
            {
                var metadata = JsonConvert.SerializeObject(result.ResponseMetadata.Metadata, new JsonSerializerSettings {
                    MaxDepth = 3
                });
                context?.Logger.LogLine($"Dynamo Call failed with {result.HttpStatusCode} {metadata}");
                context?.Logger.LogLine($"Data: {string.Join(Environment.NewLine, values.Select(s => s.ToString()))}");
                throw new Exception(metadata);
            }
        }
Пример #8
0
        public static List <BatchWriteItemResponse> WriteItemsAsync(List <Dictionary <string, AttributeValue> > items)
        {
            var resultList = new List <BatchWriteItemResponse>();

            for (int i = 0; i <= items.Count / 25; i++)
            {
                int index = i * 25;
                int count = Math.Min(25, items.Count - index);
                var batch = items.GetRange(index, count);

                System.Diagnostics.Debug.WriteLine($"Processing Items {index}-{index + count}");

                var request = new BatchWriteItemRequest();
                var writes  = new List <WriteRequest>();
                foreach (var item in batch)
                {
                    writes.Add(new WriteRequest(new PutRequest()
                    {
                        Item = item
                    }));
                }
                request.RequestItems.Add(Manager.DestinationTable.Name, writes);

                var result = Manager.Client.BatchWriteItemAsync(request).Result;
                resultList.Add(result);
            }
            return(resultList);
        }
        private static async Task <BatchWriteItemResponse> BuildDelete <Item>(Item[] items, string tableName, ItemConverter <Item> itemConverter, AmazonDynamoDBClient dynamodbClient)
        {
            List <WriteRequest> writeRequests = new List <WriteRequest>();

            foreach (Item item in items)
            {
                DeleteRequest deleteRequest = new DeleteRequest()
                {
                    Key = itemConverter.GetItemAsAttributes(item)
                };

                WriteRequest writeRequest = new WriteRequest()
                {
                    DeleteRequest = deleteRequest
                };

                writeRequests.Add(writeRequest);
            }

            BatchWriteItemRequest batchWriteItemRequest = new BatchWriteItemRequest()
            {
                RequestItems = new Dictionary <string, List <WriteRequest> >()
                {
                    {
                        tableName,
                        writeRequests
                    }
                }
            };

            return(await dynamodbClient.BatchWriteItemAsync(batchWriteItemRequest));
        }
Пример #10
0
        // This processes 25 quotes at a time since the max is 25 as per dynamoDB batch write
        private async Task WriteBatch(IEnumerable <Quote> batchQuotes)
        {
            // There is this possibility when there is nothing to approve
            if (batchQuotes.Count() == 0)
            {
                Console.WriteLine("Batch empty");
                return;
            }

            Console.WriteLine("Writing batch");
            Dictionary <string, List <WriteRequest> > writeRequests = new Dictionary <string, List <WriteRequest> >();

            writeRequests.Add(DataDefinitions.QUOTES_TABLE, new List <WriteRequest>());
            foreach (Quote quote in batchQuotes)
            {
                PutRequest request = new PutRequest(QuoteExtensions.AttributesOfQuote(quote));
                writeRequests[DataDefinitions.QUOTES_TABLE].Add(new WriteRequest(request));
            }

            BatchWriteItemRequest batchRequest = new BatchWriteItemRequest
            {
                RequestItems = writeRequests
            };

            Console.WriteLine("Before writing to dynamodb");
            BatchWriteItemResponse batchResponse = await _client.BatchWriteItemAsync(batchRequest);

            Console.WriteLine($"quotes batch response: {batchResponse.HttpStatusCode}");
        }
Пример #11
0
        private BatchWriteItemRequest ConstructRequest(Dictionary <string, QuickList <WriteRequestDocument> > writeItems, Table targetTable, out Dictionary <Key, Document> documentMap, bool isAsync)
        {
            documentMap = new Dictionary <Key, Document>(keyComparer);
            BatchWriteItemRequest request = new BatchWriteItemRequest();

            foreach (var writeItem in writeItems)
            {
                string tableName = writeItem.Key;
                List <WriteRequestDocument> requestItems = writeItem.Value.GetItems();
                if (requestItems != null && requestItems.Count > 0)
                {
                    var table       = tableMap[tableName];
                    var requestList = new List <WriteRequest>();

                    foreach (var item in requestItems)
                    {
                        requestList.Add(item.WriteRequest);
                        // Add document corresponding to the Put request to document map
                        if (item.WriteRequest.PutRequest != null)
                        {
                            var key = table.MakeKey(item.Document);
                            documentMap.Add(key, item.Document);
                        }
                    }
                    request.RequestItems[tableName] = requestList;
                }
            }

            request.BeforeRequestEvent += isAsync ?
                                          new RequestEventHandler(targetTable.UserAgentRequestEventHandlerAsync) :
                                          new RequestEventHandler(targetTable.UserAgentRequestEventHandlerSync);

            return(request);
        }
Пример #12
0
        public void PutItems <T>(IEnumerable <T> items)
        {
            var table     = DynamoMetadata.GetTable <T>();
            var remaining = items.ToList();

            PopulateMissingHashes(table, remaining);

            while (remaining.Count > 0)
            {
                var batchSize = Math.Min(remaining.Count, MaxWriteBatchSize);
                var nextBatch = remaining.GetRange(0, batchSize);
                remaining.RemoveRange(0, batchSize);

                var putItems = nextBatch.Map(x => new WriteRequest(
                                                 new PutRequest(Converters.ToAttributeValues(this, x, table))));

                var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> > {
                    { table.Name, putItems }
                });

                var response = Exec(() => DynamoDb.BatchWriteItem(request));

                var i = 0;
                while (response.UnprocessedItems.Count > 0)
                {
                    response = Exec(() => DynamoDb.BatchWriteItem(new BatchWriteItemRequest(response.UnprocessedItems)));

                    if (response.UnprocessedItems.Count > 0)
                    {
                        i.SleepBackOffMultiplier();
                    }
                }
            }
        }
Пример #13
0
        /// <summary>
        /// <para>The <i>BatchWriteItem</i> operation puts or deletes multiple items in one or more tables. A single call to <i>BatchWriteItem</i> can
        /// write up to 1 MB of data, which can comprise as many as 25 put or delete requests. Individual items to be written can be as large as 64
        /// KB.</para> <para><b>NOTE:</b> BatchWriteItem cannot update items. To update items, use the UpdateItem API. </para> <para>The individual
        /// <i>PutItem</i> and <i>DeleteItem</i> operations specified in <i>BatchWriteItem</i> are atomic; however <i>BatchWriteItem</i> as a whole is
        /// not. If any requested operations fail because the table's provisioned throughput is exceeded or an internal processing failure occurs, the
        /// failed operations are returned in the <i>UnprocessedItems</i> response parameter. You can investigate and optionally resend the requests.
        /// Typically, you would call <i>BatchWriteItem</i> in a loop. Each iteration would check for unprocessed items and submit a new
        /// <i>BatchWriteItem</i> request with those unprocessed items until all items have been processed.</para> <para>To write one item, you can use
        /// the <i>PutItem</i> operation; to delete one item, you can use the <i>DeleteItem</i> operation.</para> <para>With <i>BatchWriteItem</i> , you
        /// can efficiently write or delete large amounts of data, such as from Amazon Elastic MapReduce (EMR), or copy data from another database into
        /// DynamoDB. In order to improve performance with these large-scale operations, <i>BatchWriteItem</i> does not behave in the same way as
        /// individual <i>PutItem</i> and <i>DeleteItem</i> calls would For example, you cannot specify conditions on individual put and delete
        /// requests, and <i>BatchWriteItem</i> does not return deleted items in the response.</para> <para>If you use a programming language that
        /// supports concurrency, such as Java, you can use threads to write items in parallel. Your application must include the necessary logic to
        /// manage the threads.</para> <para>With languages that don't support threading, such as PHP, <i>BatchWriteItem</i> will write or delete the
        /// specified items one at a time. In both situations, <i>BatchWriteItem</i> provides an alternative where the API performs the specified put
        /// and delete operations in parallel, giving you the power of the thread pool approach without having to introduce complexity into your
        /// application.</para> <para>Parallel processing reduces latency, but each specified put and delete request consumes the same number of write
        /// capacity units whether it is processed in parallel or not. Delete operations on nonexistent items consume one write capacity unit.</para>
        /// <para>If one or more of the following is true, DynamoDB rejects the entire batch write operation:</para>
        /// <ul>
        /// <li> <para>One or more tables specified in the <i>BatchWriteItem</i> request does not exist.</para> </li>
        /// <li> <para>Primary key attributes specified on an item in the request do not match those in the corresponding table's primary key
        /// schema.</para> </li>
        /// <li> <para>You try to perform multiple operations on the same item in the same <i>BatchWriteItem</i> request. For example, you cannot put
        /// and delete the same item in the same <i>BatchWriteItem</i> request. </para> </li>
        /// <li> <para>The total request size exceeds 1 MB.</para> </li>
        /// <li> <para>Any individual item in a batch exceeds 64 KB.</para> </li>
        ///
        /// </ul>
        /// </summary>
        ///
        /// <param name="batchWriteItemRequest">Container for the necessary parameters to execute the BatchWriteItem service method on
        /// AmazonDynamoDBv2.</param>
        ///
        /// <returns>The response from the BatchWriteItem service method, as returned by AmazonDynamoDBv2.</returns>
        ///
        /// <exception cref="T:Amazon.DynamoDBv2.Model.ItemCollectionSizeLimitExceededException" />
        /// <exception cref="T:Amazon.DynamoDBv2.Model.ResourceNotFoundException" />
        /// <exception cref="T:Amazon.DynamoDBv2.Model.ProvisionedThroughputExceededException" />
        /// <exception cref="T:Amazon.DynamoDBv2.Model.InternalServerErrorException" />
        /// <param name="cancellationToken">
        ///     A cancellation token that can be used by other objects or threads to receive notice of cancellation.
        /// </param>
        public Task <BatchWriteItemResponse> BatchWriteItemAsync(BatchWriteItemRequest batchWriteItemRequest, CancellationToken cancellationToken = default(CancellationToken))
        {
            var marshaller   = new BatchWriteItemRequestMarshaller();
            var unmarshaller = BatchWriteItemResponseUnmarshaller.GetInstance();

            return(Invoke <IRequest, BatchWriteItemRequest, BatchWriteItemResponse>(batchWriteItemRequest, marshaller, unmarshaller, signer, cancellationToken));
        }
        public async Task DeletePhoto(PhotoId photoId)
        {
            logWriter.LogInformation($"{nameof(DeletePhoto)}({nameof(photoId)} = '{photoId}'");

            // Tirst we need to get the keys for all items to delete
            IEnumerable <Dictionary <string, AttributeValue> > itemKeys = await GetAllItemKeys(photoId);

            var request = new BatchWriteItemRequest
            {
                RequestItems = new Dictionary <string, List <WriteRequest> >
                {
                    {
                        tableName,
                        itemKeys.Distinct(new DbKeyEqualityComparer()).Select(key => new WriteRequest {
                            DeleteRequest = new DeleteRequest
                            {
                                Key = key
                            }
                        }).ToList()
                    }
                }
            };

            await dynamoDbCore.BatchWriteItem(request);
        }
        /// <summary>
        /// Initiates the asynchronous execution of the BatchWriteItem operation.
        /// </summary>
        ///
        /// <param name="request">Container for the necessary parameters to execute the BatchWriteItem operation.</param>
        /// <param name="cancellationToken">
        ///     A cancellation token that can be used by other objects or threads to receive notice of cancellation.
        /// </param>
        /// <returns>The task object representing the asynchronous operation.</returns>
        public Task <BatchWriteItemResponse> BatchWriteItemAsync(BatchWriteItemRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
        {
            var marshaller   = new BatchWriteItemRequestMarshaller();
            var unmarshaller = BatchWriteItemResponseUnmarshaller.Instance;

            return(InvokeAsync <BatchWriteItemRequest, BatchWriteItemResponse>(request, marshaller,
                                                                               unmarshaller, cancellationToken));
        }
Пример #16
0
 private void CallUntilCompletion(BatchWriteItemRequest request, AmazonDynamoDB client)
 {
     do
     {
         var batchWriteItemResponse = client.BatchWriteItem(request);
         var result = batchWriteItemResponse.BatchWriteItemResult;
         request.RequestItems = result.UnprocessedItems;
     } while (request.RequestItems.Count > 0);
 }
 public async Task DynamoDbUpdateCacheVideoDataAsync(List <WriteRequest> writeRequests)
 {
     var batchPutRequest = new BatchWriteItemRequest {
         RequestItems = new Dictionary <string, List <WriteRequest> > {
             { _dynamoDbVideoTableName, writeRequests }
         }
     };
     await _dynamoDbClient.BatchWriteItemAsync(batchPutRequest);
 }
Пример #18
0
        private static BatchWriteItemRequest GetNextBatch(BatchWriteItemRequest request, int skip)
        {
            var writeRequests = request.RequestItems.Skip(skip).Take(MaxNumberOfBatchWriteRequestItems).ToDictionary(x => x.Key, x => x.Value);

            return(writeRequests.Any()
                ? new BatchWriteItemRequest {
                RequestItems = writeRequests, ReturnConsumedCapacity = request.ReturnConsumedCapacity
            }
                : null);
        }
Пример #19
0
        private async Task CallUntilCompletionAsync(BatchWriteItemRequest request, Dictionary <string, Dictionary <Key, Document> > documentMap, IAmazonDynamoDB client, CancellationToken cancellationToken)
#endif
        {
            do
            {
                var result = await client.BatchWriteItemAsync(request, cancellationToken).ConfigureAwait(false);

                request.RequestItems = result.UnprocessedItems;

                Dictionary <Key, Document> unprocessedDocuments = new Dictionary <Key, Document>(keyComparer);
                foreach (var unprocessedItems in result.UnprocessedItems)
                {
                    string tableName = unprocessedItems.Key;
                    Table  table     = tableMap[tableName];
                    Dictionary <Key, Document> tableDocumentMap = documentMap[tableName];

                    foreach (var writeRequest in unprocessedItems.Value)
                    {
                        if (writeRequest.PutRequest != null)
                        {
                            var doc = table.FromAttributeMap(writeRequest.PutRequest.Item);
                            var key = table.MakeKey(doc);

                            Document document = null;
                            if (tableDocumentMap.TryGetValue(key, out document))
                            {
                                // Remove unprocessed requests from the document map
                                // and copy them to unprocessed documents.
                                unprocessedDocuments.Add(key, document);
                                tableDocumentMap.Remove(key);
                            }
                        }
                    }

                    // Commit the remaining documents in the document map
                    foreach (var document in tableDocumentMap.Values)
                    {
                        document.CommitChanges();
                    }
                    // Replace existing documents with just the unprocessed documents
                    documentMap[tableName] = unprocessedDocuments;
                }
            } while (request.RequestItems.Count > 0);

            // Commit any remaining documents in document map.
            // This would only happen if we are not able to match the items sent in the request
            // with the items returned back as unprocessed items.
            foreach (var tableDocumentMap in documentMap.Values)
            {
                foreach (var document in tableDocumentMap.Values)
                {
                    document.CommitChanges();
                }
            }
        }
Пример #20
0
        private static int GetNumberOfWrites(BatchWriteItemRequest request)
        {
            int totalWrites = 0;

            foreach (var writeItem in request.RequestItems)
            {
                var writes = writeItem.Value;
                totalWrites += (writes == null ? 0 : writes.Count);
            }
            return(totalWrites);
        }
Пример #21
0
        public async Task SaveAsync(List <ScheduledTask> scheduledTasks)
        {
            var request = new BatchWriteItemRequest()
            {
                RequestItems = new Dictionary <string, List <WriteRequest> >()
                {
                    { Constants.ScheduledTasks, GetWriteItems(scheduledTasks) }
                }
            };

            await _dbClient.BatchWriteItemAsync(request);
        }
        private BatchWriteItemRequest GetHashtagsRequests(string text, IEnumerable <HashtagModel> hashtags, PhotoModel existingItem, HashtagUpdateMode updateMode)
        {
            IEnumerable <HashtagModel> hashtagsBefore = updateMode == HashtagUpdateMode.Update
                ? existingItem.Hashtags
                : emptyHashtagSequence;

            existingItem.RawText  = text;
            existingItem.Hashtags = hashtags;

            BatchWriteItemRequest writeItemsRequest = GetHashtagsWriteRequests(hashtags, existingItem, hashtagsBefore, updateMode);

            return(writeItemsRequest);
        }
        /// <summary>
        /// Removes items from a DynamoDB table.
        /// </summary>
        /// <param name="client">An initialized DynamoDB client object.</param>
        /// <param name="tableName">The name of the table from which the item
        /// will be deleted.</param>
        /// <param name="idstring">The id of the item to be deleted.</param>
        /// <param name="area">The area of the item to delete.</param>
        /// <returns>The response from the call to BatchWriteItemAsync.</returns>
        public static async Task <BatchWriteItemResponse> RemoveItemsAsync(
            IAmazonDynamoDB client,
            string tableName,
            string idstring,
            string area)
        {
            var ids = idstring.Split(" ");

            var writeRequests = new List <WriteRequest>();
            var items         = new Dictionary <string, List <WriteRequest> >();

            for (int i = 0; i < ids.Length; i++)
            {
                var writeRequest = new WriteRequest
                {
                    // For the operation to delete an item, if you provide a primary key value
                    // that does not exist in the table, there is no error.
                    DeleteRequest = new DeleteRequest
                    {
                        Key = new Dictionary <string, AttributeValue>()
                        {
                            {
                                "ID",
                                new AttributeValue
                                {
                                    S = ids[i],
                                }
                            },
                            {
                                "Area",
                                new AttributeValue
                                {
                                    S = area,
                                }
                            },
                        },
                    },
                };

                writeRequests.Add(writeRequest);
            }

            items.Add(tableName, writeRequests);

            var request = new BatchWriteItemRequest(items);

            var response = await client.BatchWriteItemAsync(request);

            return(response);
        }
Пример #24
0
        private async Task WriteSeries(int?rowCount, bool runContinuous, bool useGroupedTable, string seriesKey)
        {
            using (var client = new AmazonDynamoDBClient(credentials, config))
            {
                var dataSet = new DataSet(seriesKey);

                var writeToTableName = useGroupedTable
                    ? groupedTableName
                    : tableName;

                var records = useGroupedTable
                    ? dataSet.GetRowGroups(rowCount, runContinuous).Select(CreateWriteRequest)
                    : dataSet.GetRows(rowCount, runContinuous).Select(CreateWriteRequest);

                var count = 0;

                foreach (var batch in records.Batch(25)) // DynamoDB supports writing 25 records at a time
                {
                    var requestItems = new Dictionary <string, List <WriteRequest> >
                    {
                        [writeToTableName] = batch
                    };

                    while (requestItems.Any())
                    {
                        var request = new BatchWriteItemRequest()
                        {
                            RequestItems = requestItems
                        };

                        try
                        {
                            var response = await client.BatchWriteItemAsync(request);

                            // Attempt to write unprocessed items again
                            requestItems = response.UnprocessedItems;
                        }
                        catch (AmazonDynamoDBException ex)
                        {
                            await output.WriteLineAsync(ex.Message);
                        }
                    }

                    count += batch.Count;
                    Interlocked.Add(ref totalCount, batch.Count);

                    await output.WriteLineAsync($"Wrote {count} records from dataset '{dataSet.SeriesKey}' to: {writeToTableName}");
                }
            }
        }
Пример #25
0
        internal BatchWriteItemResponse BatchWriteItem(BatchWriteItemRequest request)
        {
            var task = BatchWriteItemAsync(request);

            try
            {
                return(task.Result);
            }
            catch (AggregateException e)
            {
                ExceptionDispatchInfo.Capture(e.InnerException).Throw();
                return(null);
            }
        }
Пример #26
0
        /// <summary>
        /// Initiates the asynchronous execution of the BatchWriteItem operation.
        /// <seealso cref="Amazon.DynamoDBv2.IAmazonDynamoDB"/>
        /// </summary>
        /// <param name="request">Container for the necessary parameters to execute the BatchWriteItem operation.</param>
        /// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes</param>
        /// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
        ///          procedure using the AsyncState property.</param>
        /// <returns>void</returns>
        public void BatchWriteItemAsync(BatchWriteItemRequest request, AmazonServiceCallback callback, object state)
        {
            if (!AmazonInitializer.IsInitialized)
            {
                throw new Exception("AWSPrefab is not added to the scene");
            }

            ThreadPool.QueueUserWorkItem(new WaitCallback(delegate
            {
                var marshaller   = new BatchWriteItemRequestMarshaller();
                var unmarshaller = BatchWriteItemResponseUnmarshaller.Instance;
                Invoke(request, callback, state, marshaller, unmarshaller, signer);
            }));
            return;
        }
Пример #27
0
        /// <summary>
        /// Sends a list of write requests in batches as large as the AWS SDK will allow.
        /// </summary>
        /// <param name="client">the client</param>
        /// <param name="tableName">the table name</param>
        /// <param name="requests">list of requests</param>
        /// <returns>async Task with no return value</returns>
        public static async Task BatchWriteRequestsAsync(AmazonDynamoDBClient client, string tableName,
                                                         List <WriteRequest> requests)
        {
            int batchSize = 25;

            for (int i = 0; i < requests.Count; i += batchSize)
            {
                var batch   = requests.GetRange(i, Math.Min(batchSize, requests.Count - i));
                var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> >()
                {
                    { tableName, batch }
                });
                await client.BatchWriteItemAsync(request);
            }
        }
Пример #28
0
        private void CallUntilCompletion(BatchWriteItemRequest request, Dictionary <Key, Document> documentMap, AmazonDynamoDB client)
        {
            do
            {
                var batchWriteItemResponse = client.BatchWriteItem(request);
                var result = batchWriteItemResponse.BatchWriteItemResult;
                request.RequestItems = result.UnprocessedItems;

                Dictionary <Key, Document> unprocessedDocuments = new Dictionary <Key, Document>(keyComparer);
                foreach (var unprocessedItems in result.UnprocessedItems)
                {
                    Table table = tableMap[unprocessedItems.Key];

                    foreach (var writeRequest in unprocessedItems.Value)
                    {
                        if (writeRequest.PutRequest != null)
                        {
                            var key = table.MakeKey(Document.FromAttributeMap(writeRequest.PutRequest.Item));

                            Document document = null;
                            if (documentMap.TryGetValue(key, out document))
                            {
                                // Remove unprocessed requests from the document map
                                // and copy them to unprocessed documents.
                                unprocessedDocuments.Add(key, document);
                                documentMap.Remove(key);
                            }
                        }
                    }
                }

                // Commit the remaining documents in the document map
                foreach (var document in documentMap.Values)
                {
                    document.CommitChanges();
                }
                documentMap = unprocessedDocuments;
            } while (request.RequestItems.Count > 0);

            // Commit any remaining documents in document map.
            // This would only happen if we are not able to match the items sent in the request
            // with the items returned back as unprocessed items.
            foreach (var document in documentMap.Values)
            {
                document.CommitChanges();
            }
        }
Пример #29
0
        private List <string> WriteBigBatch(string hashTableName, int items, int itemsStartingIndex, int itemSize)
        {
            List <string> itemIds  = new List <string>();
            string        itemData = new string('@', itemSize);

            List <WriteRequest> writeRequests = new List <WriteRequest>();

            for (int i = 0; i < items; i++)
            {
                var itemId = (itemsStartingIndex + i).ToString();
                itemIds.Add(itemId);

                var writeRequest = new WriteRequest
                {
                    PutRequest = new PutRequest
                    {
                        Item = new Dictionary <string, AttributeValue>
                        {
                            { "Id", new AttributeValue {
                                  N = itemId
                              } },
                            { "Data", new AttributeValue {
                                  S = itemData
                              } }
                        }
                    }
                };
                writeRequests.Add(writeRequest);
            }

            var request = new BatchWriteItemRequest
            {
                RequestItems = new Dictionary <string, List <WriteRequest> >
                {
                    { hashTableName, writeRequests }
                }
            };
            BatchWriteItemResult result;

            do
            {
                result = Client.BatchWriteItem(request);
                request.RequestItems = result.UnprocessedItems;
            } while (result.UnprocessedItems != null && result.UnprocessedItems.Count > 0);

            return(itemIds);
        }
        private Task BatchDeleteAsync(IEnumerable <string> messageIds)
        {
            var requests = messageIds.Select(x =>
                                             new WriteRequest(
                                                 new DeleteRequest(
                                                     new Dictionary <string, AttributeValue> {
                { MESSAGE_ID, new AttributeValue(x) }
            }
                                                     )
                                                 )
                                             ).ToList();
            var request = new BatchWriteItemRequest(new Dictionary <string, List <WriteRequest> > {
                { _tableName, requests }
            });

            return(_dynamoClient.BatchWriteItemAsync(request));
        }