/// <summary> /// Insere uma coleção de entidades na tabela no Azure Table Storage. /// </summary> public async Task <TableBatchResult> BatchInsertOrMergeEntityAsync(CloudTable table, IList <T> entities) { try { if (entities != null) { TableBatchOperation tableBatchOperation = new TableBatchOperation(); foreach (var item in entities) { tableBatchOperation.InsertOrMerge(item); } TableBatchResult tableBachResult = await table.ExecuteBatchAsync(tableBatchOperation); return(tableBachResult); } else { throw new ArgumentNullException("entities"); } } catch (Exception) { throw; } }
/// <summary> /// Note: This method fails when change-set is bigger than 100 entries (maximal batch size) /// </summary> public async Task <int> Synchronize(Func <T, bool> predicate, ICollection <T> entities) { try { var existingKeys = _table.CreateQuery <T>().Where(predicate).Select(x => new { x.PartitionKey, x.RowKey }).ToArray(); var created = 0; var updated = 0; var deleted = 0; var batchOperation = new TableBatchOperation(); foreach (var entity in entities.Where(e => !existingKeys.Any(x => x.PartitionKey == e.PartitionKey && x.RowKey == e.RowKey))) { batchOperation.Add(TableOperation.InsertOrReplace(entity)); created++; } foreach (var entity in entities.Where(e => existingKeys.Any(x => x.PartitionKey == e.PartitionKey && x.RowKey == e.RowKey))) { batchOperation.Add(TableOperation.InsertOrReplace(entity)); updated++; } foreach (var keyPair in existingKeys.Where(x => !entities.Any(e => x.PartitionKey == e.PartitionKey && x.RowKey == e.RowKey))) { var entity = await GetAsync(keyPair.PartitionKey, keyPair.RowKey); batchOperation.Add(TableOperation.Delete(entity)); deleted++; } TableBatchResult result = null; if (batchOperation.Count > 0) { result = await _table.ExecuteBatchAsync(batchOperation); } return(created + updated + deleted); } catch (StorageException) { throw; } }
public async Task <IEnumerable <IAggregatedEvent> > FetchTimeSeriesAggregatedEventsAsync(IEnumerable <int> artistWorkIds, TimeSeries timeSeries) { List <TimeSeriesValueEntity> entities = new List <TimeSeriesValueEntity>(); if (!artistWorkIds.Any()) { var query = new TableQuery <TimeSeriesValueEntity>().Where( TableQuery.GenerateFilterCondition( nameof(TimeSeriesValueEntity.TimeSeries), QueryComparisons.Equal, timeSeries.ToString()) ); entities = _cacheTable.ExecuteQuery(query).ToList(); } else { TableBatchOperation batchOperation = new TableBatchOperation(); foreach (var idBatch in artistWorkIds.Batch(50)) { foreach (var id in idBatch) { batchOperation.Retrieve <TimeSeriesValueEntity>(id.ToString(), timeSeries.ToString()); } TableBatchResult batchResult = await _cacheTable.ExecuteBatchAsync(batchOperation); entities.AddRange(batchResult.Select(x => (TimeSeriesValueEntity)x.Result)); batchOperation.Clear(); } } return(entities.Select(entity => { return new AggregatedEvent() { AggregatedEventSum = entity.TimeSeriesTotal, AggregationTimeSeries = entity.TimeSeries, Id = entity.ArtistWorkId, AggregatedEventSumSource = entity.TimeSeriesValues.Select(x => new AggregatedEventSource() { Timestamp = x.Timestamp, Value = x.Value }) }; })); }
public async Task <IEnumerable <TableResult> > ExecuteBatchAsync(CloudTable table) { ConcurrentBag <TableResult> results = new ConcurrentBag <TableResult>(); IEnumerable <Task> tasks = _batches.Select((batchOperation) => { return(table.ExecuteBatchAsync(batchOperation) .ContinueWith((taskTableBatch) => { TableBatchResult batchResult = taskTableBatch.Result; foreach (var tr in batchResult) { results.Add(tr); } })); }); await Task.WhenAll(tasks).ConfigureAwait(false); Clear(); return(results as IEnumerable <TableResult>); }
public TableBatchResult ExecuteBatch(TableBatchOperation tableOp, string tableName) { if (tableOp.Count > 0) //Cannot execute an empty batch { try { CloudTable table = tableClient.GetTableReference(tableName); TableBatchResult result = table.ExecuteBatch(tableOp); return(result); } catch (Exception e) { throw; } } return(new TableBatchResult()); }
internal static RESTCommand <TableBatchResult> GenerateCMDForTableBatchOperation(TableBatchOperation batch, CloudTableClient client, CloudTable table, TableRequestOptions requestOptions) { RESTCommand <TableBatchResult> rESTCommand = new RESTCommand <TableBatchResult>(client.Credentials, client.StorageUri); RESTCommandGeneratorUtils.ApplyTableRequestOptionsToStorageCommand(requestOptions, rESTCommand); rESTCommand.HttpClient = client.HttpClient; TableBatchResult results = new TableBatchResult(); rESTCommand.CommandLocationMode = ((!batch.ContainsWrites) ? CommandLocationMode.PrimaryOrSecondary : CommandLocationMode.PrimaryOnly); rESTCommand.CommandLocationMode = CommandLocationMode.PrimaryOnly; rESTCommand.ParseErrorAsync = StorageExtendedErrorInformationRestHelper.ReadExtendedErrorInfoFromStreamAsync; rESTCommand.BuildRequest = ((RESTCommand <TableBatchResult> cmd, Uri uri, UriQueryBuilder builder, HttpContent httpContent, int?timeout, OperationContext ctx) => TableRequestMessageFactory.BuildStorageRequestMessageForTableBatchOperation(uri, batch, SharedKeyCanonicalizer.Instance, table.Name, client.Credentials, ctx, requestOptions)); rESTCommand.PreProcessResponse = ((RESTCommand <TableBatchResult> cmd, HttpResponseMessage resp, Exception ex, OperationContext ctx) => HttpResponseParsers.ProcessExpectedStatusCodeNoException(HttpStatusCode.Accepted, resp?.StatusCode ?? HttpStatusCode.Unused, results, cmd, ex)); rESTCommand.PostProcessResponseAsync = ((RESTCommand <TableBatchResult> cmd, HttpResponseMessage resp, OperationContext ctx, CancellationToken token) => TableOperationHttpResponseParsers.TableBatchOperationPostProcessAsync(results, batch, cmd, resp, ctx, requestOptions, client.Credentials.AccountName, token)); rESTCommand.RecoveryAction = delegate { results.Clear(); }; return(rESTCommand); }
/// <summary> /// Demonstrate inserting of a large batch of entities. Some considerations for batch operations: /// 1. You can perform updates, deletes, and inserts in the same single batch operation. /// 2. A single batch operation can include up to 100 entities. /// 3. All entities in a single batch operation must have the same partition key. /// 4. While it is possible to perform a query as a batch operation, it must be the only operation in the batch. /// 5. Batch size must be less than or equal to 2 MB /// </summary> /// <param name="table">Sample table name</param> /// <param name="partitionKey">The partition for the entity</param> /// <returns>A Task object</returns> private static async Task BatchInsertOfCustomerEntitiesAsync(CloudTable table, string partitionKey) { try { // Create the batch operation. TableBatchOperation batchOperation = new TableBatchOperation(); // The following code generates test data for use during the query samples. for (int i = 0; i < 100; i++) { batchOperation.InsertOrMerge(new CustomerEntity(partitionKey, string.Format("{0}", i.ToString("D4"))) { Email = string.Format("{0}@contoso.com", i.ToString("D4")), PhoneNumber = string.Format("425-555-{0}", i.ToString("D4")) }); } // Execute the batch operation. TableBatchResult results = await table.ExecuteBatchAsync(batchOperation); foreach (var res in results) { var customerInserted = res.Result as CustomerEntity; Console.WriteLine("Inserted entity with\t Etag = {0} and PartitionKey = {1}, RowKey = {2}", customerInserted.ETag, customerInserted.PartitionKey, customerInserted.RowKey); } if (results.RequestCharge.HasValue) { Console.WriteLine("Request Charge of the Batch Operation against Cosmos DB Table: " + results.RequestCharge); } } catch (StorageException e) { Console.WriteLine(e.Message); Console.ReadLine(); throw; } }
/// <summary> /// Note: This method fails when change-set is bigger than 100 entries (maximal batch size) /// </summary> public async Task <int> Synchronize(Func <T, bool> predicate, ICollection <T> entities) { try { var existingKeys = _table.CreateQuery <T>().Where(predicate).Select(x => new { x.PartitionKey, x.RowKey }).ToArray(); var created = 0; var updated = 0; var deleted = 0; var batchOperation = new TableBatchOperation(); foreach (var entity in entities.Where(e => !existingKeys.Any(x => x.PartitionKey == e.PartitionKey && x.RowKey == e.RowKey))) { batchOperation.Add(TableOperation.InsertOrReplace(entity)); created++; } foreach (var entity in entities.Where(e => existingKeys.Any(x => x.PartitionKey == e.PartitionKey && x.RowKey == e.RowKey))) { batchOperation.Add(TableOperation.InsertOrReplace(entity)); updated++; } foreach (var keyPair in existingKeys.Where(x => !entities.Any(e => x.PartitionKey == e.PartitionKey && x.RowKey == e.RowKey))) { var entity = await GetAsync(keyPair.PartitionKey, keyPair.RowKey); batchOperation.Add(TableOperation.Delete(entity)); deleted++; } TableBatchResult result = null; if (batchOperation.Count > 0) { result = await _table.ExecuteBatchAsync(batchOperation); } if (created > 0) { var metric = string.Format(Constants.Metrics.EntityCreatedPattern, typeof(T).Name); _log.LogMetric(metric, created); } if (updated > 0) { var metric = string.Format(Constants.Metrics.EntityUpdatedPattern, typeof(T).Name); _log.LogMetric(metric, updated); } if (deleted > 0) { var metric = string.Format(Constants.Metrics.EntityDeletedPattern, typeof(T).Name); _log.LogMetric(metric, deleted); } return(created + updated + deleted); } catch (StorageException e) { _log.LogError(e, $"Unable to access data in table '{_table.Name}'"); throw; } }
public static void EnsureSuccessStatusCode(this TableBatchResult batchResult) { batchResult.ForEach(x => x.EnsureSuccessStatusCode()); }
public static bool HasSuccessStatusCode(this TableBatchResult batchResult) { return(batchResult.All(x => x.HasSuccessStatusCode())); }
internal static async Task <TableBatchResult> TableBatchOperationPostProcessAsync(TableBatchResult result, TableBatchOperation batch, RESTCommand <TableBatchResult> cmd, HttpResponseMessage resp, OperationContext ctx, TableRequestOptions options, string accountName, CancellationToken cancellationToken) { Stream responseStream = cmd.ResponseStream; StreamReader streamReader = new StreamReader(responseStream); await streamReader.ReadLineAsync().ConfigureAwait(continueOnCapturedContext: false); string currentLine3 = await streamReader.ReadLineAsync().ConfigureAwait(continueOnCapturedContext: false); int index = 0; bool failError = false; while (currentLine3 != null && !currentLine3.StartsWith("--batchresponse")) { while (!currentLine3.StartsWith("HTTP")) { currentLine3 = await streamReader.ReadLineAsync().ConfigureAwait(continueOnCapturedContext: false); } int statusCode = int.Parse(currentLine3.Substring(9, 3)); Dictionary <string, string> headers = new Dictionary <string, string>(); currentLine3 = await streamReader.ReadLineAsync().ConfigureAwait(continueOnCapturedContext: false); while (!string.IsNullOrWhiteSpace(currentLine3)) { int num = currentLine3.IndexOf(':'); headers[currentLine3.Substring(0, num)] = currentLine3.Substring(num + 2); currentLine3 = await streamReader.ReadLineAsync().ConfigureAwait(continueOnCapturedContext: false); } MemoryStream bodyStream = null; currentLine3 = await streamReader.ReadLineAsync().ConfigureAwait(continueOnCapturedContext: false); if (statusCode != 204) { bodyStream = new MemoryStream(Encoding.UTF8.GetBytes(currentLine3)); } await streamReader.ReadLineAsync().ConfigureAwait(continueOnCapturedContext: false); currentLine3 = await streamReader.ReadLineAsync().ConfigureAwait(continueOnCapturedContext: false); TableOperation tableOperation = batch[index]; TableResult obj = new TableResult { Result = tableOperation.Entity }; result.Add(obj); string arg = null; if (headers.ContainsKey("Content-Type")) { arg = headers["Content-Type"]; } obj.HttpStatusCode = statusCode; bool flag; if (tableOperation.OperationType == TableOperationType.Insert) { failError = (statusCode == 409); flag = ((!tableOperation.EchoContent) ? (statusCode != 204) : (statusCode != 201)); } else if (tableOperation.OperationType == TableOperationType.Retrieve) { if (statusCode == 404) { index++; continue; } flag = (statusCode != 200); } else { failError = (statusCode == 404); flag = (statusCode != 204); } if (failError) { if (cmd.ParseErrorAsync != null) { cmd.CurrentResult.ExtendedErrorInformation = cmd.ParseErrorAsync(bodyStream, resp, arg, CancellationToken.None).Result; } cmd.CurrentResult.HttpStatusCode = statusCode; if (!string.IsNullOrEmpty(cmd.CurrentResult.ExtendedErrorInformation.ErrorMessage)) { string errorMessage = cmd.CurrentResult.ExtendedErrorInformation.ErrorMessage; cmd.CurrentResult.HttpStatusMessage = errorMessage.Substring(0, errorMessage.IndexOf("\n", StringComparison.Ordinal)); } else { cmd.CurrentResult.HttpStatusMessage = statusCode.ToString(CultureInfo.InvariantCulture); } throw new StorageException(cmd.CurrentResult, (cmd.CurrentResult.ExtendedErrorInformation != null) ? cmd.CurrentResult.ExtendedErrorInformation.ErrorMessage : "An unknown error has occurred, extended error information not available.", null) { IsRetryable = false }; } if (flag) { if (cmd.ParseErrorAsync != null) { cmd.CurrentResult.ExtendedErrorInformation = cmd.ParseErrorAsync(bodyStream, resp, arg, CancellationToken.None).Result; } cmd.CurrentResult.HttpStatusCode = statusCode; if (!string.IsNullOrEmpty(cmd.CurrentResult.ExtendedErrorInformation.ErrorMessage)) { string errorMessage2 = cmd.CurrentResult.ExtendedErrorInformation.ErrorMessage; cmd.CurrentResult.HttpStatusMessage = errorMessage2.Substring(0, errorMessage2.IndexOf("\n", StringComparison.Ordinal)); } else { cmd.CurrentResult.HttpStatusMessage = statusCode.ToString(CultureInfo.InvariantCulture); } string arg2 = Convert.ToString(index, CultureInfo.InvariantCulture); if (cmd.CurrentResult.ExtendedErrorInformation != null && !string.IsNullOrEmpty(cmd.CurrentResult.ExtendedErrorInformation.ErrorMessage)) { string text = ExtractEntityIndexFromExtendedErrorInformation(cmd.CurrentResult); if (!string.IsNullOrEmpty(text)) { arg2 = text; } } throw new StorageException(cmd.CurrentResult, string.Format(CultureInfo.CurrentCulture, "Element {0} in the batch returned an unexpected response code.", arg2), null) { IsRetryable = true }; } if (headers.ContainsKey("ETag") && !string.IsNullOrEmpty(headers["ETag"])) { obj.Etag = headers["ETag"]; if (tableOperation.Entity != null) { tableOperation.Entity.ETag = obj.Etag; } } if (tableOperation.OperationType == TableOperationType.Retrieve || (tableOperation.OperationType == TableOperationType.Insert && tableOperation.EchoContent)) { if (!headers["Content-Type"].Contains("application/json;odata=nometadata")) { await ReadOdataEntityAsync(obj, tableOperation, bodyStream, ctx, accountName, options, cancellationToken).ConfigureAwait(continueOnCapturedContext: false); } else { await ReadEntityUsingJsonParserAsync(obj, tableOperation, bodyStream, ctx, options, cancellationToken).ConfigureAwait(continueOnCapturedContext: false); } } else if (tableOperation.OperationType == TableOperationType.Insert) { tableOperation.Entity.Timestamp = ParseETagForTimestamp(obj.Etag); } index++; } return(result); }
public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "ip")] HttpRequest req, [Table(HostIpDataEntity.TableName, Connection = "AzureWebJobsStorage")] CloudTable cloudTable, ClaimsPrincipal principal, ILogger log) { var userIdClaim = principal?.FindFirst(Claims.ObjectId); var userId = userIdClaim?.Value; if (string.IsNullOrWhiteSpace(userId)) { log.LogError("request has invalid userId {UserId}", userId); return(new UnauthorizedResult()); } log.LogTrace("IP update request received"); string requestBody = await new StreamReader(req.Body).ReadToEndAsync(); UpdateIPRequestModel model; try { model = JsonConvert.DeserializeObject <UpdateIPRequestModel>(requestBody); } catch (JsonException) { log.LogError("IP update request has invalid payload"); return(new BadRequestResult()); } var validationResult = new List <ValidationResult>(); bool isValid = Validator.TryValidateObject(model, new ValidationContext(model), validationResult); if (!isValid) { log.LogError("IP update request has invalid model"); return(new BadRequestObjectResult(validationResult)); } try { TableBatchOperation batchOperations = new TableBatchOperation(); foreach (var nicInfo in model.NicIp) { var entity = new HostIpDataEntity(userId, model.ComputerName, nicInfo.Id, nicInfo.Name, nicInfo.Addresses); TableOperation insertOp = TableOperation.InsertOrMerge(entity); batchOperations.Add(insertOp); } TableBatchResult result = await cloudTable.ExecuteBatchAsync(batchOperations); } catch (StorageException e) { log.LogError(e.Message); throw; } log.LogTrace("successfully updated IP for computer {ComputerName}, {Entries} entries updated", model.ComputerName, model.NicIp.Count); return(new NoContentResult()); }
public void SaveGame(Game game) { if (game is null) { throw new ArgumentNullException("game"); } try { //Game TableOperation saveGame = TableOperation.InsertOrReplace(game); TableResult saveGameResult = tableService.Execute(saveGame, "Games"); //Board (or BoardLocations) TableBatchOperation saveBoard = new TableBatchOperation(); foreach (BoardLocation bl in game.Board) { bl.PartitionKey = game.PartitionKey; bl.RowKey = bl.DisplayNumber; saveBoard.InsertOrReplace(bl); } TableBatchResult saveBoardResult = tableService.ExecuteBatch(saveBoard, "BoardLocations"); //Plays var playBatches = game.Plays.Batch(100); foreach (var batch in playBatches) { TableBatchOperation savePlays = new TableBatchOperation(); foreach (Play p in batch) { p.PartitionKey = game.PartitionKey; p.RowKey = p.TurnNumber.ToString(); savePlays.InsertOrReplace(p); } TableBatchResult savePlaysResult = tableService.ExecuteBatch(savePlays, "Plays"); } //Teams TableBatchOperation saveTeams = new TableBatchOperation(); foreach (Team t in game.Teams) { t.PartitionKey = game.PartitionKey; t.RowKey = t.TeamNumber.ToString(); saveTeams.InsertOrReplace(t); } TableBatchResult saveTeamsResult = tableService.ExecuteBatch(saveTeams, "Teams"); //Players TableBatchOperation savePlayers = new TableBatchOperation(); foreach (Player p in game.Players) { p.PartitionKey = game.PartitionKey; p.RowKey = p.PlayerNumber.ToString(); savePlayers.InsertOrReplace(p); } TableBatchResult savePlayersResult = tableService.ExecuteBatch(savePlayers, "Players"); } catch (StorageException e) { throw; } }
internal static async Task <TResult> ExecuteBatchOperationAsync <TResult>(TableBatchOperation batch, CloudTableClient client, CloudTable table, TableRequestOptions requestOptions, OperationContext operationContext, CancellationToken cancellationToken) where TResult : class { TableOperationType currentOperationType = batch.First().OperationType; batch.First(); try { List <Document> list = new List <Document>(); List <TableOperationType> list2 = new List <TableOperationType>(); List <string> list3 = new List <string>(); foreach (TableOperation item2 in batch) { currentOperationType = item2.OperationType; Document item = (item2.OperationType != TableOperationType.Retrieve) ? EntityHelpers.GetDocumentFromEntity(item2.Entity, operationContext, requestOptions) : EntityTranslator.GetDocumentWithPartitionAndRowKey(item2.RetrievePartitionKey, item2.RetrieveRowKey); list.Add(item); list2.Add(item2.OperationType); list3.Add((item2.Entity == null) ? string.Empty : EtagHelper.ConvertToBackEndETagFormat(item2.Entity.ETag)); } RequestOptions requestOptions2 = GetRequestOptions(batch.batchPartitionKey, requestOptions); Uri storedProcedureUri = UriFactory.CreateStoredProcedureUri("TablesDB", table.Name, "__.sys.tablesBatchOperation"); StoredProcedureResponse <string> storedProcedureResponse = await table.ServiceClient.DocumentClient.ExecuteStoredProcedureAsync <string>(storedProcedureUri, requestOptions2, new object[3] { list.ToArray(), list2.ToArray(), list3.ToArray() }); JArray jArray = JArray.Parse(storedProcedureResponse.Response); TableBatchResult tableBatchResult = new TableBatchResult(); tableBatchResult.RequestCharge = storedProcedureResponse.RequestCharge; for (int i = 0; i < jArray.Count; i++) { tableBatchResult.Add(GetTableResultFromDocument(batch[i], jArray[i].ToObject <Document>(), operationContext, requestOptions, storedProcedureResponse.SessionToken, 0.0)); } return(tableBatchResult as TResult); } catch (Exception ex) { DocumentClientException ex2 = ex as DocumentClientException; if (ex2 != null && ex2.StatusCode == HttpStatusCode.BadRequest && ex2.Message.Contains("Resource Not Found") && currentOperationType == TableOperationType.Retrieve) { TableBatchResult tableBatchResult2 = new TableBatchResult(); tableBatchResult2.Add(new TableResult { Etag = null, HttpStatusCode = 404, Result = null }); tableBatchResult2.RequestCharge = ex2.RequestCharge; return(tableBatchResult2 as TResult); } TableErrorResult tableErrorResult = ex.TranslateDocumentErrorForStoredProcs(null, batch.Count); RequestResult requestResult = GenerateRequestResult(tableErrorResult.ExtendedErroMessage, tableErrorResult.HttpStatusCode, tableErrorResult.ExtendedErrorCode, tableErrorResult.ExtendedErroMessage, tableErrorResult.ServiceRequestID, tableErrorResult.RequestCharge); StorageException ex3 = new StorageException(requestResult, requestResult.ExtendedErrorInformation.ErrorMessage, ex); if (ex2 != null) { PopulateOperationContextForBatchOperations(operationContext, ex3, ex2.ActivityId); } throw ex3; } }