private static async Task AcceptSuggestionInternal([NotNull] AppEntity app, [NotNull] SuggestionEntity suggestion, int retries) { var batchOperation = new TableBatchOperation { TableOperation.Delete(suggestion), TableOperation.InsertOrReplace(new ProcessedSuggestionEntity(suggestion)) }; if (suggestion.IsRetype) { batchOperation.Delete(app); batchOperation.Insert(new AppEntity(app.SteamAppId, app.SteamName, suggestion.AppType)); } else { batchOperation.Replace(app); } var table = await GetTable(SteamToHltbTableName, retries).ConfigureAwait(false); CommonEventSource.Log.AcceptSuggestionStart(suggestion.SteamAppId, suggestion.HltbId); await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false); CommonEventSource.Log.AcceptSuggestionStop(suggestion.SteamAppId, suggestion.HltbId); }
protected override TableBatchOperation PrepareNextBatchOperation(IList <T> batch, string partitionKey, out List <XTableResult> results) { // Search for existing entities versions List <VersionedTableEntity> existingVersions = FindExistingVersions(batch, partitionKey); results = new List <XTableResult>(batch.Count); TableBatchOperation batchOperation = new TableBatchOperation(); // Add data operations foreach (T entity in batch) { VersionedTableEntity existingEntity = existingVersions.FirstOrDefault(x => x.RowKey == entity.RowKey); results.Add(new XTableResult()); // Insert entity if it doesn't exist if (existingEntity == null) { batchOperation.Insert(entity); } else { entity.Version = VersionIncrementer.Increment(existingEntity.Version); // Set current ETag entity.ETag = existingEntity.ETag; batchOperation.Replace(entity); } } return(batchOperation); }
private void AddOperationToBatch(ref TableBatchOperation tableBatchOperation, TAzureTableEntity entity, SaveType batchMethodName) { switch (batchMethodName) { case SaveType.Insert: tableBatchOperation.Insert(entity); break; case SaveType.InsertOrMerge: tableBatchOperation.InsertOrMerge(entity); break; case SaveType.InsertOrReplace: tableBatchOperation.InsertOrReplace(entity); break; case SaveType.Merge: tableBatchOperation.Merge(entity); break; case SaveType.Delete: tableBatchOperation.Delete(entity); break; case SaveType.Replace: tableBatchOperation.Replace(entity); break; } }
private static void AddOperationToBatch(ref TableBatchOperation tableBatchOperation, TAzureTableEntity entity, string batchMethodName) { switch (batchMethodName) { case CtConstants.TableOpInsert: tableBatchOperation.Insert(entity); break; case CtConstants.TableOpInsertOrMerge: tableBatchOperation.InsertOrMerge(entity); break; case CtConstants.TableOpInsertOrReplace: tableBatchOperation.InsertOrReplace(entity); break; case CtConstants.TableOpMerge: tableBatchOperation.Merge(entity); break; case CtConstants.TableOpDelete: entity.ETag = "*"; tableBatchOperation.Delete(entity); break; case CtConstants.TableOpReplace: tableBatchOperation.Replace(entity); break; } }
public static async Task UpdateAsync <TPayload>( this CloudTable table, TPayload payload, KeysPair keys) where TPayload : class, new() { Ensure.ArgumentNotNull(keys, nameof(keys)); Ensure.ArgumentNotNull(payload, nameof(payload)); Ensure.ArgumentNotNull(table, nameof(table)); var entity = new DynamicTableEntity(keys.PartitionKey, keys.RowKey); entity.Timestamp = DateTime.UtcNow; entity.Properties = EntityPropertyConverter.Flatten(payload, new OperationContext()); entity.ETag = "*"; var batch = new TableBatchOperation(); batch.Replace(entity); await table.ExecuteBatchAsync(batch) .ConfigureAwait(false); await table.ExecuteBatchAsync(batch) .ConfigureAwait(false); }
protected override async Task DeleteMessagesToAsync(string persistenceId, long toSequenceNr, bool isPermanent) { try { CloudTable table = _extension.TableJournalSettings .GetClient(persistenceId) .GetTableReference(_extension.TableJournalSettings.TableName); IEnumerable <Event> results = table.ExecuteQuery(BuildDeleteTableQuery(persistenceId, toSequenceNr)) .OrderByDescending(t => t.SequenceNr); if (results.Any()) { TableBatchOperation batchOperation = new TableBatchOperation(); foreach (Event s in results) { s.IsDeleted = true; if (isPermanent) { batchOperation.Delete(s); } else { batchOperation.Replace(s); } } await table.ExecuteBatchAsync(batchOperation); } } catch (Exception ex) { _log.Error(ex, ex.Message); throw; } }
/// <summary> /// Update collection of notes for individual personal goal in storage. /// </summary> /// <param name="personalGoalNoteEntities">Holds collection of personal goal notes data.</param> /// <returns>A boolean that represents personal goal note detail entities are updated.</returns> public async Task <bool> UpdatePersonalGoalNoteDetailsAsync(IEnumerable <PersonalGoalNoteDetail> personalGoalNoteEntities) { personalGoalNoteEntities = personalGoalNoteEntities ?? throw new ArgumentNullException(nameof(personalGoalNoteEntities)); try { await this.EnsureInitializedAsync(); TableBatchOperation tableBatchOperation = new TableBatchOperation(); int batchCount = (int)Math.Ceiling((double)personalGoalNoteEntities.Count() / PersonalGoalNotesPerBatch); for (int batchCountIndex = 0; batchCountIndex < batchCount; batchCountIndex++) { var personalGoalNoteEntitiesBatch = personalGoalNoteEntities.Skip(batchCountIndex * PersonalGoalNotesPerBatch).Take(PersonalGoalNotesPerBatch); foreach (var personalGoalNoteEntity in personalGoalNoteEntitiesBatch) { tableBatchOperation.Replace(personalGoalNoteEntity); } if (tableBatchOperation.Count > 0) { await this.CloudTable.ExecuteBatchAsync(tableBatchOperation); } } return(true); } catch (Exception ex) { this.logger.LogError(ex, $"An error occurred in {nameof(this.CreateOrUpdatePersonalGoalNoteDetailsAsync)} while storing notes in storage."); throw; } }
public async Task Add(TableOperation operation) { batchOperation.Add(operation); if (operation.Entity == key) { batchContainsKey = true; } if (batchOperation.Count == AzureTableConstants.MaxBatchSize - (batchContainsKey ? 0 : 1)) { // the key serves as a synchronizer, to prevent modification by multiple grains under edge conditions, // like duplicate activations or deployments.Every batch write needs to include the key, // even if the key values don't change. if (!batchContainsKey) { if (string.IsNullOrEmpty(key.ETag)) { batchOperation.Insert(key); } else { batchOperation.Replace(key); } } await Flush().ConfigureAwait(false); batchOperation.Clear(); batchContainsKey = false; } }
// TODO: add methods for // delete single record // Delete an entity // Retrieve entities in pages asynchronously /// <summary> /// MS docs - https://docs.microsoft.com/en-us/azure/cosmos-db/table-storage-how-to-use-dotnet#retrieve-entities-in-pages-asynchronously /// </summary> // TODO: update that //https://stackoverflow.com/questions/17955557/painfully-slow-azure-table-insert-and-delete-batch-operations public void BatchUpdate <TEntity>(IList <TEntity> data, ILogger logger = null) where TEntity : ITableEntity { var groupedByPartition = data.GroupBy(x => x.PartitionKey).ToList(); foreach (var group in groupedByPartition) { var entities = group.Select(x => x).ToList(); logger?.Info("Saving for PartitionKey = " + group.Key + $" ({entities.Count}) records"); int rowOffset = 0; while (rowOffset < entities.Count) { var rows = entities.Skip(rowOffset).Take(100).ToList(); rowOffset += rows.Count; string partition = "$" + rowOffset.ToString(); var batch = new TableBatchOperation(); foreach (var row in rows) { batch.Replace(row); } CloudTable.ExecuteBatchAsync(batch); logger?.Info("Updated batch for partition " + partition); } } }
public void Replace(string partitionKey, string rowKey, string etag, string propertyName, object propertyValue) { Require.NotEmpty(partitionKey, "partitionKey"); Require.NotEmpty(rowKey, "rowKey"); Require.NotEmpty(etag, "etag"); Require.NotEmpty(propertyName, "propertyName"); AssertBatchSizeIsNotExceeded(); var entity = m_tableEntityConverter.CreateDynamicTableEntityFromProperties(propertyName, propertyValue); entity.PartitionKey = partitionKey; entity.RowKey = rowKey; entity.ETag = etag; m_batch.Replace(entity); }
public void Replace(E entity) { CheckNotSaved(); var fat = ConvertToFatEntity(entity); fat.ETag = fat.ETag ?? "*"; _context.Replace(fat); _isDirty = true; }
public void TableBatchReplaceSync() { // Insert Entity Console.WriteLine("Calling Insert()..."); DynamicReplicatedTableEntity baseEntity = new DynamicReplicatedTableEntity("replace test", "foo"); baseEntity.Properties.Add("prop1", new EntityProperty("value1")); this.repTable.Execute(TableOperation.Insert(baseEntity)); // ReplaceEntity DynamicReplicatedTableEntity replaceEntity = new DynamicReplicatedTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = "1" }; replaceEntity.Properties.Add("prop2", new EntityProperty("value2")); Console.WriteLine("Calling Replace()..."); TableBatchOperation batch = new TableBatchOperation(); batch.Replace(replaceEntity); this.repTable.ExecuteBatch(batch); // Retrieve Entity & Verify Contents Console.WriteLine("Calling Retrieve()..."); TableResult result = this.repTable.Execute(TableOperation.Retrieve <DynamicReplicatedTableEntity>(baseEntity.PartitionKey, baseEntity.RowKey)); DynamicReplicatedTableEntity retrievedEntity = result.Result as DynamicReplicatedTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]); // // Replace() again // Console.WriteLine("Calling Replace() again, setting Etag to {0}", retrievedEntity._rtable_Version); replaceEntity = new DynamicReplicatedTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = retrievedEntity._rtable_Version.ToString() }; replaceEntity.Properties.Add("prop3", new EntityProperty("value3")); batch = new TableBatchOperation(); batch.Replace(replaceEntity); this.repTable.ExecuteBatch(batch); Console.WriteLine("Calling Retrieve()..."); result = this.repTable.Execute(TableOperation.Retrieve <DynamicReplicatedTableEntity>(baseEntity.PartitionKey, baseEntity.RowKey)); retrievedEntity = result.Result as DynamicReplicatedTableEntity; Assert.IsNotNull(retrievedEntity); Console.WriteLine("{0}", retrievedEntity.ToString()); Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(replaceEntity.Properties["prop3"], retrievedEntity.Properties["prop3"]); }
public void TableBatchAllSupportedOperationsSync() { TableBatchOperation batch = new TableBatchOperation(); string pk = Guid.NewGuid().ToString(); // insert batch.Insert(GenerateRandomEnitity(pk)); // delete { DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk); this.repTable.Execute(TableOperation.Insert(entity)); batch.Delete(entity); } // replace { DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk); this.repTable.Execute(TableOperation.Insert(entity)); batch.Replace(entity); } // insert or replace { DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk); this.repTable.Execute(TableOperation.Insert(entity)); batch.InsertOrReplace(entity); } // merge { DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk); this.repTable.Execute(TableOperation.Insert(entity)); batch.Merge(entity); } // insert or merge { DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk); this.repTable.Execute(TableOperation.Insert(entity)); batch.InsertOrMerge(entity); } IList <TableResult> results = this.repTable.ExecuteBatch(batch); Assert.AreEqual(results.Count, 6); IEnumerator <TableResult> enumerator = results.GetEnumerator(); for (int i = 0; i < results.Count; i++) { enumerator.MoveNext(); Assert.AreEqual((int)HttpStatusCode.NoContent, enumerator.Current.HttpStatusCode, "HttpStatusCode mismatch i={0}", i); } }
public IList <TableResult> UpdateEntities(string tableName, IList <StorageEntity> entities) { SetProperties(entities); CloudTable table = _storage.GetTableReference(tableName); TableBatchOperation operation = new TableBatchOperation(); foreach (StorageEntity entity in entities) { operation.Replace(entity); } return(table.ExecuteBatch(operation)); }
private static async Task ReplaceTableEntities(CloudTable table, IEnumerable <ITableEntity> entities) { await Task.WhenAll( Group(entities, azureStorageBatchSize) .Select(batch => { TableBatchOperation opsBatch = new TableBatchOperation(); foreach (var item in batch) { opsBatch.Replace(item); } return(table.ExecuteBatchAsync(opsBatch)); })); }
/// <summary> /// Stores a user data element, overwriting the previous one if present. /// </summary> /// <param name="user">The user the data belongs to.</param> /// <param name="key">The key of the data element (case insensitive).</param> /// <param name="value">The value of the data element, <c>null</c> for deleting the data.</param> /// <returns><c>true</c> if the data element is stored, <c>false</c> otherwise.</returns> /// <exception cref="ArgumentNullException">If <paramref name="user"/> or <paramref name="key"/> are <c>null</c>.</exception> /// <exception cref="ArgumentException">If <paramref name="key"/> is empty.</exception> public bool StoreUserData(UserInfo user, string key, string value) { if (user == null) { throw new ArgumentNullException("user"); } if (key == null) { throw new ArgumentNullException("key"); } if (key.Length == 0) { throw new ArgumentException("Key cannot be empty", "key"); } try { // If the user does not exists return false if (GetUserEntity(_wiki, user.Username) == null) { return(false); } CloudTable table = _cloudTableClient.GetTableReference(UserDataTable); var batchOperation = new TableBatchOperation(); UserDataEntity userDataEntity = GetUserDataEntity(_wiki, user.Username, key); if (userDataEntity == null) { userDataEntity = new UserDataEntity() { PartitionKey = _wiki + "|" + user.Username, RowKey = key, Value = value }; batchOperation.Insert(userDataEntity); } else { userDataEntity.Value = value; batchOperation.Replace(userDataEntity); } table.ExecuteBatch(batchOperation); return(true); } catch (Exception ex) { throw ex; } }
public virtual async Task <IEnumerable <TableResult> > UpdateRangeAndReturnTableResultsAsync(IEnumerable <TEntity> entities) { var batchOperation = new TableBatchOperation(); foreach (var entity in entities) { var tableEntityProxy = await _tableEntityProxyFactory.BuildAsync(entity); batchOperation.Replace(tableEntityProxy); } var results = await ExecuteBatchAsLimitedBatchesAsync(batchOperation); return(results); }
public static IEnumerable <TableBatchInformation <T> > ReplaceAll <T>(this IEnumerable <IEnumerable <T> > batches) where T : ITableEntity { foreach (var batch in batches.Select(b => b.ToArray())) { var operation = new TableBatchOperation(); foreach (var instance in batch) { operation.Replace(instance); } yield return(new TableBatchInformation <T>(operation, batch, TableOperationType.Replace)); } }
public static async Task <IActionResult> MarkAsDone( [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = null)] HttpRequest req, [Table("todo")] CloudTable todoTable, ILogger log) { if (!req.Query.ContainsKey("id")) { return(new BadRequestResult()); } string uid = req.Query["id"]; Guid uuid; if (!Guid.TryParse(uid, out uuid)) { return(new NotFoundResult()); } try { var query = new TableQuery <ToDoItem>().Where( TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, uid) ); var result = await todoTable.ExecuteQuerySegmentedAsync(query, null); var elem = result.First(); elem.Done = true; elem.DoneTimestamp = (long)DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1)).TotalSeconds; var batch = new TableBatchOperation(); batch.Replace(elem); await todoTable.ExecuteBatchAsync(batch); return(new OkObjectResult(elem.ToToDoDTO())); } catch (Exception e) { // Return 404 if process fails since most of the cases fail due to invalid IDs log.LogCritical(e, "Delete Failed"); return(new NotFoundResult()); } }
public async Task <string> Persist(string expectedETag, string metadata, List <PendingTransactionState <TState> > statesToPrepare) { try { var batchOperation = new TableBatchOperation(); this.key.ETag = expectedETag; this.key.Metadata = metadata; if (string.IsNullOrEmpty(this.key.ETag)) { batchOperation.Insert(this.key); } else { batchOperation.Replace(this.key); } // add new states List <Tuple <string, long> > stored = this.states.Select(s => Tuple.Create(s.TransactionId, s.SequenceId)).ToList(); List <StateEntity> newStates = new List <StateEntity>(); foreach (PendingTransactionState <TState> pendingState in statesToPrepare.Where(p => !stored.Contains(Tuple.Create(p.TransactionId, p.SequenceId)))) { var newState = StateEntity.Create(this.jsonSettings, this.partition, pendingState); newStates.Add(newState); batchOperation.Insert(newState); } if (batchOperation.Count > AzureTableConstants.MaxBatchSize) { this.logger.LogError("Too many pending states. PendingStateCount {PendingStateCount}.", batchOperation.Count); throw new InvalidOperationException($"Too many pending states. PendingStateCount {batchOperation.Count}"); } await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false); this.states.AddRange(newStates); return(this.key.ETag); } catch (Exception ex) { this.logger.LogError("Transactional state persist failed {Exception}.", ex); throw; } }
/// <summary> /// Stores the value of a Setting. /// </summary> /// <param name="name">The name of the Setting.</param> /// <param name="value">The value of the Setting. Value cannot contain CR and LF characters, which will be removed.</param> /// <returns>True if the Setting is stored, false otherwise.</returns> /// <exception cref="ArgumentNullException">If <paramref name="name"/> is <c>null</c>.</exception> /// <exception cref="ArgumentException">If <paramref name="name"/> is empty.</exception> public bool SetSetting(string name, string value) { if (name == null) { throw new ArgumentNullException("name"); } if (name.Length == 0) { throw new ArgumentException("name"); } _settingsDictionary = null; // Nulls are converted to empty strings if (value == null) { value = ""; } GlobalSettingsEntity settingsEntity = GetGlobalSettingsEntity(name); CloudTable table = _cloudTableClient.GetTableReference(GlobalSettingsTable); var batchOperation = new TableBatchOperation(); if (settingsEntity == null) { settingsEntity = new GlobalSettingsEntity() { PartitionKey = "0", RowKey = name, Value = value }; batchOperation.Insert(settingsEntity); } else { settingsEntity.Value = value; batchOperation.Replace(settingsEntity); } table.ExecuteBatch(batchOperation); return(true); }
private static async Task DeleteSuggestionInternal([NotNull] SuggestionEntity suggestion, AppEntity app, int retries) { var batchOperation = new TableBatchOperation { TableOperation.Delete(suggestion), TableOperation.InsertOrReplace(new ProcessedSuggestionEntity(suggestion)) }; if (app != null) //this means we want to update the app to be a verified game and stop future non-game suggestions { app.VerifiedGame = true; batchOperation.Replace(app); } var table = await GetTable(SteamToHltbTableName, retries).ConfigureAwait(false); CommonEventSource.Log.DeleteSuggestionStart(suggestion.SteamAppId, suggestion.HltbId); await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false); CommonEventSource.Log.DeleteSuggestionStop(suggestion.SteamAppId, suggestion.HltbId); }
public async Task RunOrchestrator( [OrchestrationTrigger] IDurableOrchestrationContext context, [Table("population")] CloudTable storage, [Table("generation")] CloudTable generation, [Queue("population-roulette")] ICollector <string> populationRulet, ILogger log) { log.LogInformation($"Population Encounter orchestrator started"); var population = context.GetInput <IList <Population> >(); var taskList = from pop in population select context.CallActivityAsync <Population>("PupulationEncounter_FitnessFunction", pop); log.LogInformation($"Run tasks for Fitness calculation"); var encountedpopulation = new List <Population>(await Task.WhenAll(taskList.ToArray())); var populationFitnessSum = encountedpopulation.Sum(a => a.Fitness); var batchUpdateOperation = new TableBatchOperation(); log.LogInformation($"Run tasks for Adaptation calculation"); foreach (var pop in encountedpopulation) { pop.Adaptation = (pop.Fitness / populationFitnessSum); batchUpdateOperation.Replace(pop); } log.LogInformation($"Save updated data about population to DB"); storage.ExecuteBatch(batchUpdateOperation); var generationNumber = encountedpopulation.FirstOrDefault().PartitionKey; TableOperation insertOrMergeOperation = TableOperation.InsertOrMerge(new Generation() { PartitionKey = generationNumber, RowKey = generationNumber, PopulationFitness = populationFitnessSum, BestResult = encountedpopulation.Where(a => a.Adaptation == encountedpopulation.Max(a => a.Adaptation)).Select(a => a.Value.ToString()).Aggregate((a, b) => a + ", " + b) }); generation.Execute(insertOrMergeOperation); log.LogInformation("Add message to rulette queue."); populationRulet.Add(generationNumber); }
/// <summary> /// Updates the entity batch async. /// </summary> /// <typeparam name="T"></typeparam> /// <param name="entities">The entities.</param> /// <param name="entitiesToAdd">The entities to add.</param> /// <param name="entitiesToDelete">The entities to delete.</param> /// <param name="replace">if set to <c>true</c> [force].</param> /// <returns>Task{OperationResult}.</returns> public async Task <OperationResult> UpdateEntityBatchAsync <T>( IList <T> entities, IList <T> entitiesToAdd = null, IList <T> entitiesToDelete = null, bool replace = true) where T : TableEntity { var batchOperation = new TableBatchOperation(); foreach (var entity in entities) { if (replace) { batchOperation.Replace(entity); } else { batchOperation.Merge(entity); } } return(await CompleteUpdateDeleteBatchWithRestOperationsAndStart(batchOperation, entitiesToAdd, entitiesToDelete)); }
public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] ChangePasswordInfo info, ILogger log) { var table = TableUtilities.GetCloudTable("Applications"); var operation = TableOperation.Retrieve <ApplicationEntity>(info.GroupId, info.ApplicationId); var result = await table.ExecuteAsync(operation); var application = result.Result as ApplicationEntity; if (application == null) { return(new NotFoundResult()); } if (application.Password.Equals(info.Password) == false) { return(new UnauthorizedResult()); } var query = table.CreateQuery <ApplicationEntity>() .Where(p => p.PartitionKey.Equals(info.GroupId)) .AsTableQuery(); var entities = await table.ExecuteQueryAsync(query); var batch = new TableBatchOperation(); foreach (var entity in entities) { entity.Password = info.NewPassword; batch.Replace(entity); } await table.ExecuteBatchAsync(batch); return(new NoContentResult()); }
protected override TableBatchOperation PrepareNextBatchOperation(IList <DynamicTableEntity> batch, string partitionKey, out List <XTableResult> results) { // Search for existing entities versions List <VersionedTableEntity> existingVersions = FindExistingVersions(batch, partitionKey); results = new List <XTableResult>(batch.Count); TableBatchOperation batchOperation = new TableBatchOperation(); // Add data operations for (int i = 0; i < batch.Count; ++i) { VersionedTableEntity existingEntity = existingVersions.FirstOrDefault(x => x.RowKey == batch[i].RowKey); results.Add(new XTableResult()); // Insert entry if it doesn't exist if (existingEntity == null) { batchOperation.Insert(batch[i]); } else { // Replace entity only if its version is higher than existing one if (batch[i].Properties.ContainsKey(VersionColumn) && batch[i].Properties[VersionColumn].Int64Value > existingEntity.Version) { // Set current ETag batch[i].ETag = existingEntity.ETag; batchOperation.Replace(batch[i]); } else { results[i].Discarded = true; } } } return(batchOperation); }
public static void Execute <T>(this TableBatchOperation operation, TableOperationType type, T entity) where T : ITableEntity { switch (type) { case TableOperationType.Insert: operation.Insert(entity); break; case TableOperationType.Delete: operation.Delete(entity); break; case TableOperationType.Replace: operation.Replace(entity); break; case TableOperationType.Merge: operation.Merge(entity); break; case TableOperationType.InsertOrReplace: operation.InsertOrReplace(entity); break; case TableOperationType.InsertOrMerge: operation.InsertOrMerge(entity); break; case TableOperationType.Retrieve: operation.Retrieve(entity.PartitionKey, entity.RowKey); break; default: throw new Exception($"Invalid enum value {nameof(TableOperationType)} {type}"); } }
public void TableBatchAllSupportedOperationsSync() { CloudTableClient tableClient = GenerateCloudTableClient(); TableBatchOperation batch = new TableBatchOperation(); string pk = Guid.NewGuid().ToString(); // insert batch.Insert(GenerateRandomEnitity(pk)); // delete { DynamicTableEntity entity = GenerateRandomEnitity(pk); currentTable.Execute(TableOperation.Insert(entity)); batch.Delete(entity); } // replace { DynamicTableEntity entity = GenerateRandomEnitity(pk); currentTable.Execute(TableOperation.Insert(entity)); batch.Replace(entity); } // insert or replace { DynamicTableEntity entity = GenerateRandomEnitity(pk); currentTable.Execute(TableOperation.Insert(entity)); batch.InsertOrReplace(entity); } // merge { DynamicTableEntity entity = GenerateRandomEnitity(pk); currentTable.Execute(TableOperation.Insert(entity)); batch.Merge(entity); } // insert or merge { DynamicTableEntity entity = GenerateRandomEnitity(pk); currentTable.Execute(TableOperation.Insert(entity)); batch.InsertOrMerge(entity); } IList<TableResult> results = currentTable.ExecuteBatch(batch); Assert.AreEqual(results.Count, 6); IEnumerator<TableResult> enumerator = results.GetEnumerator(); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.Created); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); }
public void TableBatchReplaceFailAPM() { CloudTableClient tableClient = GenerateCloudTableClient(); // Insert Entity DynamicTableEntity baseEntity = new DynamicTableEntity("merge test", "foo"); baseEntity.Properties.Add("prop1", new EntityProperty("value1")); currentTable.Execute(TableOperation.Insert(baseEntity)); string staleEtag = baseEntity.ETag; // update entity to rev etag baseEntity.Properties["prop1"].StringValue = "updated value"; currentTable.Execute(TableOperation.Replace(baseEntity)); OperationContext opContext = new OperationContext(); try { // Attempt a merge with stale etag DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = staleEtag }; replaceEntity.Properties.Add("prop2", new EntityProperty("value2")); TableBatchOperation batch = new TableBatchOperation(); batch.Replace(replaceEntity); using (ManualResetEvent evt = new ManualResetEvent(false)) { IAsyncResult asyncRes = null; currentTable.BeginExecuteBatch(batch, null, opContext, (res) => { asyncRes = res; evt.Set(); }, null); evt.WaitOne(); currentTable.EndExecuteBatch(asyncRes); } Assert.Fail(); } catch (StorageException) { TestHelper.ValidateResponse(opContext, 1, (int)HttpStatusCode.PreconditionFailed, new string[] { "UpdateConditionNotSatisfied", "ConditionNotMet" }, new string[] { "The update condition specified in the request was not satisfied.", "The condition specified using HTTP conditional header(s) is not met." }); } // Delete Entity currentTable.Execute(TableOperation.Delete(baseEntity)); opContext = new OperationContext(); // try replacing with deleted entity try { DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = baseEntity.ETag }; replaceEntity.Properties.Add("prop2", new EntityProperty("value2")); TableBatchOperation batch = new TableBatchOperation(); batch.Replace(replaceEntity); using (ManualResetEvent evt = new ManualResetEvent(false)) { IAsyncResult asyncRes = null; currentTable.BeginExecuteBatch(batch, null, opContext, (res) => { asyncRes = res; evt.Set(); }, null); evt.WaitOne(); currentTable.EndExecuteBatch(asyncRes); } Assert.Fail(); } catch (StorageException) { TestHelper.ValidateResponse(opContext, 1, (int)HttpStatusCode.NotFound, new string[] { "ResourceNotFound" }, "The specified resource does not exist."); } }
public void TableBatchReplaceAPM() { CloudTableClient tableClient = GenerateCloudTableClient(); // Insert Entity DynamicTableEntity baseEntity = new DynamicTableEntity("merge test", "foo"); baseEntity.Properties.Add("prop1", new EntityProperty("value1")); currentTable.Execute(TableOperation.Insert(baseEntity)); // ReplaceEntity DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = baseEntity.ETag }; replaceEntity.Properties.Add("prop2", new EntityProperty("value2")); TableBatchOperation batch = new TableBatchOperation(); batch.Replace(replaceEntity); using (ManualResetEvent evt = new ManualResetEvent(false)) { IAsyncResult asyncRes = null; currentTable.BeginExecuteBatch(batch, (res) => { asyncRes = res; evt.Set(); }, null); evt.WaitOne(); currentTable.EndExecuteBatch(asyncRes); } // Retrieve Entity & Verify Contents TableResult result = currentTable.Execute(TableOperation.Retrieve(baseEntity.PartitionKey, baseEntity.RowKey)); DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]); }
public void Execute(TableBatchOperation batchOperation, TableEntity entity) { entity.ETag = "*"; // Always overwrite (ignore concurrency). batchOperation.Replace(entity); }
public void TableBatchAllSupportedOperationsSync() { TableBatchOperation batch = new TableBatchOperation(); string pk = Guid.NewGuid().ToString(); // insert batch.Insert(GenerateRandomEnitity(pk)); // delete { DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk); this.repTable.Execute(TableOperation.Insert(entity)); batch.Delete(entity); } // replace { DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk); this.repTable.Execute(TableOperation.Insert(entity)); batch.Replace(entity); } // insert or replace { DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk); this.repTable.Execute(TableOperation.Insert(entity)); batch.InsertOrReplace(entity); } // merge { DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk); this.repTable.Execute(TableOperation.Insert(entity)); batch.Merge(entity); } // insert or merge { DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk); this.repTable.Execute(TableOperation.Insert(entity)); batch.InsertOrMerge(entity); } IList<TableResult> results = this.repTable.ExecuteBatch(batch); Assert.AreEqual(results.Count, 6); IEnumerator<TableResult> enumerator = results.GetEnumerator(); for (int i = 0; i < results.Count; i++) { enumerator.MoveNext(); Assert.AreEqual((int)HttpStatusCode.NoContent, enumerator.Current.HttpStatusCode, "HttpStatusCode mismatch i={0}", i); } }
private async Task DoTableBatchAllSupportedOperationsAsync(TablePayloadFormat format) { tableClient.DefaultRequestOptions.PayloadFormat = format; TableBatchOperation batch = new TableBatchOperation(); string pk = Guid.NewGuid().ToString(); // insert batch.Insert(GenerateRandomEntity(pk)); // delete { DynamicTableEntity entity = GenerateRandomEntity(pk); await currentTable.ExecuteAsync(TableOperation.Insert(entity)); batch.Delete(entity); } // replace { DynamicTableEntity entity = GenerateRandomEntity(pk); await currentTable.ExecuteAsync(TableOperation.Insert(entity)); batch.Replace(entity); } // insert or replace { DynamicTableEntity entity = GenerateRandomEntity(pk); await currentTable.ExecuteAsync(TableOperation.Insert(entity)); batch.InsertOrReplace(entity); } // merge { DynamicTableEntity entity = GenerateRandomEntity(pk); await currentTable.ExecuteAsync(TableOperation.Insert(entity)); batch.Merge(entity); } // insert or merge { DynamicTableEntity entity = GenerateRandomEntity(pk); await currentTable.ExecuteAsync(TableOperation.Insert(entity)); batch.InsertOrMerge(entity); } IList<TableResult> results = await currentTable.ExecuteBatchAsync(batch); Assert.AreEqual(results.Count, 6); IEnumerator<TableResult> enumerator = results.GetEnumerator(); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.Created); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); }
public void TableBatchReplaceSync() { CloudTableClient tableClient = GenerateCloudTableClient(); // Insert Entity DynamicTableEntity baseEntity = new DynamicTableEntity("merge test", "foo"); baseEntity.Properties.Add("prop1", new EntityProperty("value1")); currentTable.Execute(TableOperation.Insert(baseEntity)); // ReplaceEntity DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = baseEntity.ETag }; replaceEntity.Properties.Add("prop2", new EntityProperty("value2")); TableBatchOperation batch = new TableBatchOperation(); batch.Replace(replaceEntity); currentTable.ExecuteBatch(batch); // Retrieve Entity & Verify Contents TableResult result = currentTable.Execute(TableOperation.Retrieve(baseEntity.PartitionKey, baseEntity.RowKey)); DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]); }
async Task DoRandomAtomicCalls() { for (int callNum = 0; callNum < MigrationModel.NUM_CALLS_PER_MACHINE; callNum++) { TableCall originalCall; MirrorTableCall referenceCall; SortedDictionary <PrimaryKey, DynamicTableEntity> dump = await peekProxy.DumpReferenceTableAsync(); if (PSharpRuntime.Nondeterministic()) { // Query // XXX: Test the filtering? var query = new TableQuery <DynamicTableEntity>(); query.FilterString = TableQuery.GenerateFilterCondition( TableConstants.PartitionKey, QueryComparisons.Equal, MigrationModel.SINGLE_PARTITION_KEY); // async/await pair needed to upcast the return value to object. originalCall = async table => await table.ExecuteQueryAtomicAsync(query); referenceCall = async referenceTable => await referenceTable.ExecuteQueryAtomicAsync(query); Console.WriteLine("{0} starting query", machineId); } else { // Batch write int batchSize = PSharpRuntime.Nondeterministic() ? 2 : 1; var batch = new TableBatchOperation(); var rowKeyChoices = new List <string> { "0", "1", "2", "3", "4", "5" }; for (int opNum = 0; opNum < batchSize; opNum++) { int opTypeNum = PSharpNondeterminism.Choice(7); int rowKeyI = PSharpNondeterminism.Choice(rowKeyChoices.Count); string rowKey = rowKeyChoices[rowKeyI]; rowKeyChoices.RemoveAt(rowKeyI); // Avoid duplicate in same batch var primaryKey = new PrimaryKey(MigrationModel.SINGLE_PARTITION_KEY, rowKey); string eTag = null; if (opTypeNum >= 1 && opTypeNum <= 3) { DynamicTableEntity existingEntity; int etagTypeNum = PSharpNondeterminism.Choice( dump.TryGetValue(primaryKey, out existingEntity) ? 3 : 2); switch (etagTypeNum) { case 0: eTag = ChainTable2Constants.ETAG_ANY; break; case 1: eTag = "wrong"; break; case 2: eTag = existingEntity.ETag; break; } } DynamicTableEntity entity = new DynamicTableEntity { PartitionKey = MigrationModel.SINGLE_PARTITION_KEY, RowKey = rowKey, ETag = eTag, Properties = new Dictionary <string, EntityProperty> { // Give us something to see on merge. Might help with tracing too! { string.Format("{0}_c{1}_o{2}", machineId.ToString(), callNum, opNum), new EntityProperty(true) } } }; switch (opTypeNum) { case 0: batch.Insert(entity); break; case 1: batch.Replace(entity); break; case 2: batch.Merge(entity); break; case 3: batch.Delete(entity); break; case 4: batch.InsertOrReplace(entity); break; case 5: batch.InsertOrMerge(entity); break; case 6: entity.ETag = ChainTable2Constants.ETAG_DELETE_IF_EXISTS; batch.Delete(entity); break; } } TableBatchOperation batchCopy = ChainTableUtils.CopyBatch <DynamicTableEntity>(batch); originalCall = async table => await table.ExecuteBatchAsync(batch); referenceCall = async referenceTable => await referenceTable.ExecuteMirrorBatchAsync(batchCopy, successfulBatchResult); Console.WriteLine("{0} starting batch {1}", machineId, batch); } await RunCallAsync(originalCall, referenceCall); Console.WriteLine("{0} table call verified"); } }
private async Task DoTableBatchReplaceAsync(TablePayloadFormat format) { tableClient.DefaultRequestOptions.PayloadFormat = format; // Insert Entity DynamicTableEntity baseEntity = new DynamicTableEntity("merge test", "foo" + format.ToString()); baseEntity.Properties.Add("prop1", new EntityProperty("value1")); await currentTable.ExecuteAsync(TableOperation.Insert(baseEntity)); // ReplaceEntity DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = baseEntity.ETag }; replaceEntity.Properties.Add("prop2", new EntityProperty("value2")); TableBatchOperation batch = new TableBatchOperation(); batch.Replace(replaceEntity); await currentTable.ExecuteBatchAsync(batch); // Retrieve Entity & Verify Contents TableResult result = await currentTable.ExecuteAsync(TableOperation.Retrieve(baseEntity.PartitionKey, baseEntity.RowKey)); DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]); }
private void DoEscapeTest(string data, bool useBatch, bool includeKey) { DynamicTableEntity ent = new DynamicTableEntity(includeKey ? "temp" + data : "temp", Guid.NewGuid().ToString()); ent.Properties.Add("foo", new EntityProperty(data)); // Insert if (useBatch) { TableBatchOperation batch = new TableBatchOperation(); batch.Insert(ent); currentTable.ExecuteBatch(batch); } else { currentTable.Execute(TableOperation.Insert(ent)); } // Retrieve TableResult res = null; if (useBatch) { TableBatchOperation batch = new TableBatchOperation(); batch.Retrieve(ent.PartitionKey, ent.RowKey); res = (currentTable.ExecuteBatch(batch))[0]; } else { res = currentTable.Execute(TableOperation.Retrieve(ent.PartitionKey, ent.RowKey)); } // Check equality DynamicTableEntity retrievedEntity = res.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey); Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey); Assert.AreEqual(ent.ETag, retrievedEntity.ETag); Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]); // Query using data filter TableQuery query = new TableQuery(); query.Where(string.Format( "(PartitionKey eq \'{0}\') and (RowKey eq \'{1}\') and (foo eq \'{2}\')", ent.PartitionKey, ent.RowKey, data.Replace("\'", "\'\'"))); retrievedEntity = currentTable.ExecuteQuery(query).Single(); Assert.IsNotNull(retrievedEntity); Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey); Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey); Assert.AreEqual(ent.ETag, retrievedEntity.ETag); Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]); // Merge ent.Properties.Add("foo2", new EntityProperty("bar2")); if (useBatch) { TableBatchOperation batch = new TableBatchOperation(); batch.Merge(ent); currentTable.ExecuteBatch(batch); } else { currentTable.Execute(TableOperation.Merge(ent)); } // Retrieve if (useBatch) { TableBatchOperation batch = new TableBatchOperation(); batch.Retrieve(ent.PartitionKey, ent.RowKey); res = (currentTable.ExecuteBatch(batch))[0]; } else { res = currentTable.Execute(TableOperation.Retrieve(ent.PartitionKey, ent.RowKey)); } retrievedEntity = res.Result as DynamicTableEntity; Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey); Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey); Assert.AreEqual(ent.ETag, retrievedEntity.ETag); Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]); // Replace ent.Properties.Remove("foo2"); ent.Properties.Add("foo3", new EntityProperty("bar3")); if (useBatch) { TableBatchOperation batch = new TableBatchOperation(); batch.Replace(ent); currentTable.ExecuteBatch(batch); } else { currentTable.Execute(TableOperation.Replace(ent)); } // Retrieve if (useBatch) { TableBatchOperation batch = new TableBatchOperation(); batch.Retrieve(ent.PartitionKey, ent.RowKey); res = (currentTable.ExecuteBatch(batch))[0]; } else { res = currentTable.Execute(TableOperation.Retrieve(ent.PartitionKey, ent.RowKey)); } retrievedEntity = res.Result as DynamicTableEntity; Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey); Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey); Assert.AreEqual(ent.ETag, retrievedEntity.ETag); Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]); }
async Task DoRandomAtomicCalls() { for (int callNum = 0; callNum < MigrationModel.NUM_CALLS_PER_MACHINE; callNum++) { SortedDictionary<PrimaryKey, DynamicTableEntity> dump = await peekProxy.DumpReferenceTableAsync(); if (PSharpRuntime.Nondeterministic()) { // Query var query = new TableQuery<DynamicTableEntity>(); query.FilterString = ChainTableUtils.CombineFilters( TableQuery.GenerateFilterCondition( TableConstants.PartitionKey, QueryComparisons.Equal, MigrationModel.SINGLE_PARTITION_KEY), TableOperators.And, NondeterministicUserPropertyFilterString()); await RunQueryAtomicAsync(query); } else { // Batch write int batchSize = PSharpRuntime.Nondeterministic() ? 2 : 1; var batch = new TableBatchOperation(); var rowKeyChoices = new List<string> { "0", "1", "2", "3", "4", "5" }; for (int opNum = 0; opNum < batchSize; opNum++) { int opTypeNum = PSharpNondeterminism.Choice(7); int rowKeyI = PSharpNondeterminism.Choice(rowKeyChoices.Count); string rowKey = rowKeyChoices[rowKeyI]; rowKeyChoices.RemoveAt(rowKeyI); // Avoid duplicate in same batch var primaryKey = new PrimaryKey(MigrationModel.SINGLE_PARTITION_KEY, rowKey); string eTag = null; if (opTypeNum >= 1 && opTypeNum <= 3) { DynamicTableEntity existingEntity; int etagTypeNum = PSharpNondeterminism.Choice( dump.TryGetValue(primaryKey, out existingEntity) ? 3 : 2); switch (etagTypeNum) { case 0: eTag = ChainTable2Constants.ETAG_ANY; break; case 1: eTag = "wrong"; break; case 2: eTag = existingEntity.ETag; break; } } DynamicTableEntity entity = new DynamicTableEntity { PartitionKey = MigrationModel.SINGLE_PARTITION_KEY, RowKey = rowKey, ETag = eTag, Properties = new Dictionary<string, EntityProperty> { // Give us something to see on merge. Might help with tracing too! { string.Format("{0}_c{1}_o{2}", machineId.ToString(), callNum, opNum), new EntityProperty(true) }, // Property with 50%/50% distribution for use in filters. { "isHappy", new EntityProperty(PSharpRuntime.Nondeterministic()) } } }; switch (opTypeNum) { case 0: batch.Insert(entity); break; case 1: batch.Replace(entity); break; case 2: batch.Merge(entity); break; case 3: batch.Delete(entity); break; case 4: batch.InsertOrReplace(entity); break; case 5: batch.InsertOrMerge(entity); break; case 6: entity.ETag = ChainTable2Constants.ETAG_DELETE_IF_EXISTS; batch.Delete(entity); break; } } await RunBatchAsync(batch); } } }
public void TableBatchReplaceSync() { // Insert Entity Console.WriteLine("Calling Insert()..."); DynamicReplicatedTableEntity baseEntity = new DynamicReplicatedTableEntity("replace test", "foo"); baseEntity.Properties.Add("prop1", new EntityProperty("value1")); this.repTable.Execute(TableOperation.Insert(baseEntity)); // ReplaceEntity DynamicReplicatedTableEntity replaceEntity = new DynamicReplicatedTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = "1" }; replaceEntity.Properties.Add("prop2", new EntityProperty("value2")); Console.WriteLine("Calling Replace()..."); TableBatchOperation batch = new TableBatchOperation(); batch.Replace(replaceEntity); this.repTable.ExecuteBatch(batch); // Retrieve Entity & Verify Contents Console.WriteLine("Calling Retrieve()..."); TableResult result = this.repTable.Execute(TableOperation.Retrieve<DynamicReplicatedTableEntity>(baseEntity.PartitionKey, baseEntity.RowKey)); DynamicReplicatedTableEntity retrievedEntity = result.Result as DynamicReplicatedTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]); // // Replace() again // Console.WriteLine("Calling Replace() again, setting Etag to {0}", retrievedEntity._rtable_Version); replaceEntity = new DynamicReplicatedTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = retrievedEntity._rtable_Version.ToString() }; replaceEntity.Properties.Add("prop3", new EntityProperty("value3")); batch = new TableBatchOperation(); batch.Replace(replaceEntity); this.repTable.ExecuteBatch(batch); Console.WriteLine("Calling Retrieve()..."); result = this.repTable.Execute(TableOperation.Retrieve<DynamicReplicatedTableEntity>(baseEntity.PartitionKey, baseEntity.RowKey)); retrievedEntity = result.Result as DynamicReplicatedTableEntity; Assert.IsNotNull(retrievedEntity); Console.WriteLine("{0}", retrievedEntity.ToString()); Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(replaceEntity.Properties["prop3"], retrievedEntity.Properties["prop3"]); }
public void TableBatchAllSupportedOperationsAPM() { CloudTableClient tableClient = GenerateCloudTableClient(); TableBatchOperation batch = new TableBatchOperation(); string pk = Guid.NewGuid().ToString(); // insert batch.Insert(GenerateRandomEnitity(pk)); // delete { DynamicTableEntity entity = GenerateRandomEnitity(pk); currentTable.Execute(TableOperation.Insert(entity)); batch.Delete(entity); } // replace { DynamicTableEntity entity = GenerateRandomEnitity(pk); currentTable.Execute(TableOperation.Insert(entity)); batch.Replace(entity); } // insert or replace { DynamicTableEntity entity = GenerateRandomEnitity(pk); currentTable.Execute(TableOperation.Insert(entity)); batch.InsertOrReplace(entity); } // merge { DynamicTableEntity entity = GenerateRandomEnitity(pk); currentTable.Execute(TableOperation.Insert(entity)); batch.Merge(entity); } // insert or merge { DynamicTableEntity entity = GenerateRandomEnitity(pk); currentTable.Execute(TableOperation.Insert(entity)); batch.InsertOrMerge(entity); } IList<TableResult> results = null; using (ManualResetEvent evt = new ManualResetEvent(false)) { IAsyncResult asyncRes = null; currentTable.BeginExecuteBatch(batch, (res) => { asyncRes = res; evt.Set(); }, null); evt.WaitOne(); results = currentTable.EndExecuteBatch(asyncRes); } Assert.AreEqual(results.Count, 6); IEnumerator<TableResult> enumerator = results.GetEnumerator(); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.Created); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); enumerator.MoveNext(); Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent); }
/// <summary> /// Helper function to execute the specified BatchOperation after HttpMangler is turned off and validate correctness. /// </summary> /// <param name="count">Number of operations in the batch</param> /// <param name="partitionKey">partitionKey to operate on</param> /// <param name="jobType">partitionKey is generated from jobType</param> /// <param name="jobId">RowKey is generated from jobId. JobIdTemplate = jobId-{0}</param> /// <param name="opTypes">Specifies the batch operation to be performed</param> protected void ExecuteBatchOperationAndValidate( int count, string partitionKey, string jobType, string jobId, List<TableOperationType> opTypes) { Console.WriteLine("\nExecuteBatchOperationAndValidate(): Trying to batch update {0} entities...", count); Assert.IsNotNull(opTypes, "opTypes = null"); Assert.AreEqual(count, opTypes.Count, "count and opTypes.Count should be the same"); string jobIdTemplate = jobId + "-{0}"; string replaceMessageTemplate = "updated-after-httpMangler-{0}"; IEnumerable<SampleRTableEntity> allEntities = null; TableBatchOperation batchOperation = null; int m = 0; bool gotExceptionInLastAttempt = true; int retries = 0; while (retries < MaxRetries) { try { // // GetAllRows() // allEntities = this.rtableWrapper.GetAllRows(partitionKey); // // Create a batchOperation to perform the specified opTypes // batchOperation = new TableBatchOperation(); m = 0; foreach (SampleRTableEntity entity in allEntities) { if (opTypes[m] == TableOperationType.Replace) { // set up the new entity to be used in the batch operation SampleRTableEntity replaceEntity = new SampleRTableEntity( entity.JobType, entity.JobId, string.Format(replaceMessageTemplate, m)) { ETag = entity._rtable_Version.ToString() }; // add the operation to the batch operation batchOperation.Replace(replaceEntity); } else if (opTypes[m] == TableOperationType.Delete) { batchOperation.Delete(entity); } else { throw new ArgumentException( string.Format("opType={0} is NOT supported", opTypes[m]), "opType"); } m++; } // // Call this.repTable.ExecuteBatch(batchOperation); // if (batchOperation.Count == 0) { Console.WriteLine("retries={0}. Done. batchOperation.Count == 0", retries); } else { this.repTable.ExecuteBatch(batchOperation); Console.WriteLine("retries={0}. Done ExecuteBatch()", retries); } gotExceptionInLastAttempt = false; break; } catch (RTableConflictException ex) { Console.WriteLine("retries={0}. ExecuteBatch() got an RTableConflictException: {1}", retries, ex.ToString()); retries++; gotExceptionInLastAttempt = true; Thread.Sleep(ConflictExceptionSleepTimeInMsec); } } Console.WriteLine("gotExceptionInLastAttempt={0} retries={1} MaxRetries={2}", gotExceptionInLastAttempt, retries, MaxRetries); Assert.IsFalse(gotExceptionInLastAttempt, "The last API call should not throw an exception."); // // Final validation // Console.WriteLine("Final validation..."); allEntities = this.rtableWrapper.GetAllRows(partitionKey); m = 0; int opTypesCounter = 0; foreach (SampleRTableEntity entity in allEntities) { Console.WriteLine("{0}", entity.ToString()); Console.WriteLine("---------------------------------------"); // If the operation is Delete, then skip it. No need to validate. while (opTypesCounter < count && opTypes[m] == TableOperationType.Delete) { m++; } Assert.IsTrue(m < count, "m={0} count={1}: m shoud be < count, but it is not.", m, count); if (opTypes[m] == TableOperationType.Replace) { Assert.AreEqual(string.Format(jobType, m), entity.JobType, "JobType does not match"); Assert.AreEqual(string.Format(jobIdTemplate, m), entity.JobId, "JobId does not match"); Assert.AreEqual(string.Format(replaceMessageTemplate, m), entity.Message, "Message does not match"); m++; } else { throw new ArgumentException( string.Format("opType={0} is NOT supported", opTypes[opTypesCounter]), "opType"); } } for (int i = 0; i < count; i++) { this.ReadFromIndividualAccountsDirectly( jobType, string.Format(jobIdTemplate, i), true); } Console.WriteLine("Passed final validation."); }
public void TableBatchOperationsWithEmptyKeys() { CloudTableClient tableClient = GenerateCloudTableClient(); // Insert Entity DynamicTableEntity ent = new DynamicTableEntity() { PartitionKey = "", RowKey = "" }; ent.Properties.Add("foo2", new EntityProperty("bar2")); ent.Properties.Add("foo", new EntityProperty("bar")); TableBatchOperation batch = new TableBatchOperation(); batch.Insert(ent); currentTable.ExecuteBatch(batch); // Retrieve Entity TableBatchOperation retrieveBatch = new TableBatchOperation(); retrieveBatch.Retrieve(ent.PartitionKey, ent.RowKey); TableResult result = currentTable.ExecuteBatch(retrieveBatch).First(); DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey); Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey); Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(ent.Properties["foo"].StringValue, retrievedEntity.Properties["foo"].StringValue); Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]); Assert.AreEqual(ent.Properties["foo2"].StringValue, retrievedEntity.Properties["foo2"].StringValue); Assert.AreEqual(ent.Properties["foo2"], retrievedEntity.Properties["foo2"]); // InsertOrMerge DynamicTableEntity insertOrMergeEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey); insertOrMergeEntity.Properties.Add("foo3", new EntityProperty("value")); batch = new TableBatchOperation(); batch.InsertOrMerge(insertOrMergeEntity); currentTable.ExecuteBatch(batch); result = currentTable.ExecuteBatch(retrieveBatch).First(); retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(insertOrMergeEntity.Properties["foo3"], retrievedEntity.Properties["foo3"]); // InsertOrReplace DynamicTableEntity insertOrReplaceEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey); insertOrReplaceEntity.Properties.Add("prop2", new EntityProperty("otherValue")); batch = new TableBatchOperation(); batch.InsertOrReplace(insertOrReplaceEntity); currentTable.ExecuteBatch(batch); result = currentTable.ExecuteBatch(retrieveBatch).First(); retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(1, retrievedEntity.Properties.Count); Assert.AreEqual(insertOrReplaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]); // Merge DynamicTableEntity mergeEntity = new DynamicTableEntity(retrievedEntity.PartitionKey, retrievedEntity.RowKey) { ETag = retrievedEntity.ETag }; mergeEntity.Properties.Add("mergeProp", new EntityProperty("merged")); batch = new TableBatchOperation(); batch.Merge(mergeEntity); currentTable.ExecuteBatch(batch); // Retrieve Entity & Verify Contents result = currentTable.ExecuteBatch(retrieveBatch).First(); retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(mergeEntity.Properties["mergeProp"], retrievedEntity.Properties["mergeProp"]); // Replace DynamicTableEntity replaceEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey) { ETag = retrievedEntity.ETag }; replaceEntity.Properties.Add("replaceProp", new EntityProperty("replace")); batch = new TableBatchOperation(); batch.Replace(replaceEntity); currentTable.ExecuteBatch(batch); // Retrieve Entity & Verify Contents result = currentTable.ExecuteBatch(retrieveBatch).First(); retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(replaceEntity.Properties["replaceProp"], retrievedEntity.Properties["replaceProp"]); // Delete Entity batch = new TableBatchOperation(); batch.Delete(retrievedEntity); currentTable.ExecuteBatch(batch); // Retrieve Entity result = currentTable.ExecuteBatch(retrieveBatch).First(); Assert.IsNull(result.Result); }
public void BatchOperationExceptionWhenUsingSmallerViewId() { long currentViewId = 100; long badViewId = currentViewId - 1; this.UpdateConfiguration(replicas, 0, false, currentViewId); string jobType = "jobType-BatchOperationExceptionWhenUsingSmallerViewId"; string jobId = "jobId-BatchOperationExceptionWhenUsingSmallerViewId"; int count = 3; // number of operations in the batch _rtable_Operation List<TableOperationType> opTypes = new List<TableOperationType>() { TableOperationType.Replace, TableOperationType.InsertOrReplace, TableOperationType.Delete, }; // // Insert // string jobIdTemplate = jobId + "-{0}"; string messageTemplate = "message-{0}"; string updatedMessageTemplate = "updated-" + messageTemplate; string partitionKey = string.Empty; // // Insert entities // for (int i = 0; i < count; i++) { SampleRTableEntity originalEntity = new SampleRTableEntity( jobType, string.Format(jobIdTemplate, i), string.Format(messageTemplate, i)); this.repTable.Execute(TableOperation.Insert(originalEntity)); partitionKey = originalEntity.PartitionKey; } // // Retrieve entities and use them to create batchOperation to Replace or Delete // IEnumerable<SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey); TableBatchOperation batchOperation = new TableBatchOperation(); int m = 0; foreach (SampleRTableEntity entity in allEntities) { Console.WriteLine("{0}", entity.ToString()); Console.WriteLine("---------------------------------------"); if (opTypes[m] == TableOperationType.Replace) { SampleRTableEntity replaceEntity = new SampleRTableEntity( entity.JobType, entity.JobId, string.Format(updatedMessageTemplate, m)) { ETag = entity._rtable_Version.ToString() }; batchOperation.Replace(replaceEntity); } else if (opTypes[m] == TableOperationType.InsertOrReplace) { SampleRTableEntity replaceEntity = new SampleRTableEntity( entity.JobType, entity.JobId, string.Format(updatedMessageTemplate, m)) { ETag = entity._rtable_Version.ToString() }; batchOperation.InsertOrReplace(replaceEntity); } else if (opTypes[m] == TableOperationType.Delete) { entity.ETag = entity._rtable_Version.ToString(); batchOperation.Delete(entity); } else { throw new ArgumentException( string.Format("opType={0} is NOT supported", opTypes[m]), "opType"); } m++; } // // Call ModifyConfigurationBlob to change the viewId of the wrapper to an older value // Console.WriteLine("Changing the viewId to badViewId {0}", badViewId); this.UpdateConfiguration(replicas, 0, false, badViewId); // // Execute Batch _rtable_Operation with bad viewId // Console.WriteLine("\nCalling BatchOperation with badViewId..."); try { this.repTable.ExecuteBatch(batchOperation); } catch (ReplicatedTableStaleViewException ex) { Console.WriteLine("Get this RTableStaleViewException: {0}", ex.Message); Assert.IsTrue(ex.Message.Contains(string.Format("current _rtable_ViewId {0} is smaller than", badViewId)), "Got unexpected exception message"); } }
public async Task TableBatchOnSecondaryAsync() { AssertSecondaryEndpoint(); CloudTable table = GenerateCloudTableClient().GetTableReference(GenerateRandomTableName()); TableRequestOptions options = new TableRequestOptions() { LocationMode = LocationMode.SecondaryOnly, RetryPolicy = new NoRetry(), }; TableBatchOperation batch = new TableBatchOperation(); batch.Retrieve("PartitionKey", "RowKey"); OperationContext context = new OperationContext(); await table.ExecuteBatchAsync(batch, options, context); Assert.AreEqual(StorageLocation.Secondary, context.LastResult.TargetLocation); batch = new TableBatchOperation(); batch.Insert(new DynamicTableEntity("PartitionKey", "RowKey")); StorageException e = await TestHelper.ExpectedExceptionAsync<StorageException>( async () => await table.ExecuteBatchAsync(batch, options, null), "Batch operations other than retrieve should not be sent to secondary"); Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message); batch = new TableBatchOperation(); batch.InsertOrMerge(new DynamicTableEntity("PartitionKey", "RowKey")); e = await TestHelper.ExpectedExceptionAsync<StorageException>( async () => await table.ExecuteBatchAsync(batch, options, null), "Batch operations other than retrieve should not be sent to secondary"); Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message); batch = new TableBatchOperation(); batch.InsertOrReplace(new DynamicTableEntity("PartitionKey", "RowKey")); e = await TestHelper.ExpectedExceptionAsync<StorageException>( async () => await table.ExecuteBatchAsync(batch, options, null), "Batch operations other than retrieve should not be sent to secondary"); Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message); batch = new TableBatchOperation(); batch.Merge(new DynamicTableEntity("PartitionKey", "RowKey") { ETag = "*" }); e = await TestHelper.ExpectedExceptionAsync<StorageException>( async () => await table.ExecuteBatchAsync(batch, options, null), "Batch operations other than retrieve should not be sent to secondary"); Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message); batch = new TableBatchOperation(); batch.Replace(new DynamicTableEntity("PartitionKey", "RowKey") { ETag = "*" }); e = await TestHelper.ExpectedExceptionAsync<StorageException>( async () => await table.ExecuteBatchAsync(batch, options, null), "Batch operations other than retrieve should not be sent to secondary"); Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message); batch = new TableBatchOperation(); batch.Delete(new DynamicTableEntity("PartitionKey", "RowKey") { ETag = "*" }); e = await TestHelper.ExpectedExceptionAsync<StorageException>( async () => await table.ExecuteBatchAsync(batch, options, null), "Batch operations other than retrieve should not be sent to secondary"); Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message); }
protected override async Task DeleteMessagesToAsync(string persistenceId, long toSequenceNr, bool isPermanent) { try { CloudTable table = _extension.TableJournalSettings .GetClient(persistenceId) .GetTableReference(_extension.TableJournalSettings.TableName); IEnumerable<Event> results = table.ExecuteQuery(BuildDeleteTableQuery(persistenceId, toSequenceNr)) .OrderByDescending(t => t.SequenceNr); if (results.Any()) { TableBatchOperation batchOperation = new TableBatchOperation(); foreach (Event s in results) { s.IsDeleted = true; if(isPermanent) { batchOperation.Delete(s); } else { batchOperation.Replace(s); } } await table.ExecuteBatchAsync(batchOperation); } } catch (Exception ex) { _log.Error(ex, ex.Message); throw; } }
private async Task DoTableBatchReplaceFailAsync(TablePayloadFormat format) { tableClient.DefaultRequestOptions.PayloadFormat = format; // Insert Entity DynamicTableEntity baseEntity = new DynamicTableEntity("merge test", "foo" + format.ToString()); baseEntity.Properties.Add("prop1", new EntityProperty("value1")); await currentTable.ExecuteAsync(TableOperation.Insert(baseEntity)); string staleEtag = baseEntity.ETag; // update entity to rev etag baseEntity.Properties["prop1"].StringValue = "updated value"; await currentTable.ExecuteAsync(TableOperation.Replace(baseEntity)); OperationContext opContext = new OperationContext(); try { // Attempt a merge with stale etag DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = staleEtag }; replaceEntity.Properties.Add("prop2", new EntityProperty("value2")); TableBatchOperation batch = new TableBatchOperation(); batch.Replace(replaceEntity); await currentTable.ExecuteBatchAsync(batch, null, opContext); Assert.Fail(); } catch (Exception) { TestHelper.ValidateResponse(opContext, 1, (int)HttpStatusCode.PreconditionFailed, new string[] { "UpdateConditionNotSatisfied", "ConditionNotMet" }, new string[] { "The update condition specified in the request was not satisfied.", "The condition specified using HTTP conditional header(s) is not met." }); } // Delete Entity await currentTable.ExecuteAsync(TableOperation.Delete(baseEntity)); opContext = new OperationContext(); // try replacing with deleted entity try { DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = baseEntity.ETag }; replaceEntity.Properties.Add("prop2", new EntityProperty("value2")); TableBatchOperation batch = new TableBatchOperation(); batch.Replace(replaceEntity); await currentTable.ExecuteBatchAsync(batch, null, opContext); Assert.Fail(); } catch (Exception) { TestHelper.ValidateResponse(opContext, 1, (int)HttpStatusCode.NotFound, new string[] { "ResourceNotFound" }, "The specified resource does not exist."); } }
private async Task DoEscapeTestAsync(string data, bool useBatch, bool includeKey) { DynamicTableEntity ent = new DynamicTableEntity(includeKey ? "temp" + data : "temp", Guid.NewGuid().ToString()); ent.Properties.Add("foo", new EntityProperty(data)); // Insert if (useBatch) { TableBatchOperation batch = new TableBatchOperation(); batch.Insert(ent); await currentTable.ExecuteBatchAsync(batch); } else { await currentTable.ExecuteAsync(TableOperation.Insert(ent)); } // Retrieve TableResult res = null; if (useBatch) { TableBatchOperation batch = new TableBatchOperation(); batch.Retrieve(ent.PartitionKey, ent.RowKey); res = (await currentTable.ExecuteBatchAsync(batch))[0]; } else { res = await currentTable.ExecuteAsync(TableOperation.Retrieve(ent.PartitionKey, ent.RowKey)); } // Check equality DynamicTableEntity retrievedEntity = res.Result as DynamicTableEntity; Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey); Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey); Assert.AreEqual(ent.ETag, retrievedEntity.ETag); Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]); // Merge ent.Properties.Add("foo2", new EntityProperty("bar2")); if (useBatch) { TableBatchOperation batch = new TableBatchOperation(); batch.Merge(ent); await currentTable.ExecuteBatchAsync(batch); } else { await currentTable.ExecuteAsync(TableOperation.Merge(ent)); } // Retrieve if (useBatch) { TableBatchOperation batch = new TableBatchOperation(); batch.Retrieve(ent.PartitionKey, ent.RowKey); res = (await currentTable.ExecuteBatchAsync(batch))[0]; } else { res = await currentTable.ExecuteAsync(TableOperation.Retrieve(ent.PartitionKey, ent.RowKey)); } retrievedEntity = res.Result as DynamicTableEntity; Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey); Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey); Assert.AreEqual(ent.ETag, retrievedEntity.ETag); Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]); // Replace ent.Properties.Remove("foo2"); ent.Properties.Add("foo3", new EntityProperty("bar3")); if (useBatch) { TableBatchOperation batch = new TableBatchOperation(); batch.Replace(ent); await currentTable.ExecuteBatchAsync(batch); } else { await currentTable.ExecuteAsync(TableOperation.Replace(ent)); } // Retrieve if (useBatch) { TableBatchOperation batch = new TableBatchOperation(); batch.Retrieve(ent.PartitionKey, ent.RowKey); res = (await currentTable.ExecuteBatchAsync(batch))[0]; } else { res = await currentTable.ExecuteAsync(TableOperation.Retrieve(ent.PartitionKey, ent.RowKey)); } retrievedEntity = res.Result as DynamicTableEntity; Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey); Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey); Assert.AreEqual(ent.ETag, retrievedEntity.ETag); Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]); }
private async Task DoTableBatchOperationsWithEmptyKeysAsync(TablePayloadFormat format) { tableClient.DefaultRequestOptions.PayloadFormat = format; // Insert Entity DynamicTableEntity ent = new DynamicTableEntity() { PartitionKey = "", RowKey = "" }; ent.Properties.Add("foo2", new EntityProperty("bar2")); ent.Properties.Add("foo", new EntityProperty("bar")); TableBatchOperation batch = new TableBatchOperation(); batch.Insert(ent); await currentTable.ExecuteBatchAsync(batch); // Retrieve Entity TableBatchOperation retrieveBatch = new TableBatchOperation(); retrieveBatch.Retrieve(ent.PartitionKey, ent.RowKey); TableResult result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First(); DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey); Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey); Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(ent.Properties["foo"].StringValue, retrievedEntity.Properties["foo"].StringValue); Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]); Assert.AreEqual(ent.Properties["foo2"].StringValue, retrievedEntity.Properties["foo2"].StringValue); Assert.AreEqual(ent.Properties["foo2"], retrievedEntity.Properties["foo2"]); // InsertOrMerge DynamicTableEntity insertOrMergeEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey); insertOrMergeEntity.Properties.Add("foo3", new EntityProperty("value")); batch = new TableBatchOperation(); batch.InsertOrMerge(insertOrMergeEntity); await currentTable.ExecuteBatchAsync(batch); result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First(); retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(insertOrMergeEntity.Properties["foo3"], retrievedEntity.Properties["foo3"]); // InsertOrReplace DynamicTableEntity insertOrReplaceEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey); insertOrReplaceEntity.Properties.Add("prop2", new EntityProperty("otherValue")); batch = new TableBatchOperation(); batch.InsertOrReplace(insertOrReplaceEntity); await currentTable.ExecuteBatchAsync(batch); result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First(); retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(1, retrievedEntity.Properties.Count); Assert.AreEqual(insertOrReplaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]); // Merge DynamicTableEntity mergeEntity = new DynamicTableEntity(retrievedEntity.PartitionKey, retrievedEntity.RowKey) { ETag = retrievedEntity.ETag }; mergeEntity.Properties.Add("mergeProp", new EntityProperty("merged")); batch = new TableBatchOperation(); batch.Merge(mergeEntity); await currentTable.ExecuteBatchAsync(batch); // Retrieve Entity & Verify Contents result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First(); retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(mergeEntity.Properties["mergeProp"], retrievedEntity.Properties["mergeProp"]); // Replace DynamicTableEntity replaceEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey) { ETag = retrievedEntity.ETag }; replaceEntity.Properties.Add("replaceProp", new EntityProperty("replace")); batch = new TableBatchOperation(); batch.Replace(replaceEntity); await currentTable.ExecuteBatchAsync(batch); // Retrieve Entity & Verify Contents result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First(); retrievedEntity = result.Result as DynamicTableEntity; Assert.IsNotNull(retrievedEntity); Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count); Assert.AreEqual(replaceEntity.Properties["replaceProp"], retrievedEntity.Properties["replaceProp"]); // Delete Entity batch = new TableBatchOperation(); batch.Delete(retrievedEntity); await currentTable.ExecuteBatchAsync(batch); // Retrieve Entity result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First(); Assert.IsNull(result.Result); }
/// <summary> /// Helper function to create some initial entities and then call the specified batchOperation with HttpMangler enabled. /// </summary> /// <param name="count"></param> /// <param name="jobType"></param> /// <param name="jobId"></param> /// <param name="targetStorageAccount"></param> /// <param name="opTypes"></param> /// <param name="targetApiExpectedToFail"></param> /// <param name="checkOriginalEntityUnchanged"></param> /// <param name="checkStorageAccountsConsistent"></param> /// <param name="httpManglerStartTime"></param> /// <param name="skipInitialSessions"></param> /// <returns>ParitionKey of the initial entities created</returns> protected string SetupAndRunTemperBatchOperation( int count, string jobType, string jobId, int targetStorageAccount, List<TableOperationType> opTypes, bool targetApiExpectedToFail, bool checkOriginalEntityUnchanged, bool checkStorageAccountsConsistent, out DateTime httpManglerStartTime, int skipInitialSessions = 0) { Assert.IsTrue(0 <= targetStorageAccount && targetStorageAccount < this.actualStorageAccountsUsed.Count, "SetupAndRunTemperBatchOperation() is called with out-of-range targetStorageAccount={0}", targetStorageAccount); int index = this.actualStorageAccountsUsed[targetStorageAccount]; string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[index]; Console.WriteLine("SetupAndRunTemperBatchOperation(): accountNameToTamper={0} skipInitialSessions={1}", accountNameToTamper, skipInitialSessions); Assert.AreEqual(count, opTypes.Count, "count and opTypes.Count should be the same"); // // Tamper behavior // ProxyBehavior[] behaviors = new[] { TamperBehaviors.TamperAllRequestsIf( (session => { session.Abort();}), skipInitialSessions, AzureStorageSelectors.TableTraffic().IfHostNameContains(accountNameToTamper + ".")) }; string jobIdTemplate = jobId + "-{0}"; string messageTemplate = "message-{0}"; string updatedMessageTemplate = "updated-" + messageTemplate; string partitionKey = string.Empty; // // Insert entities // for (int i = 0; i < count; i++) { SampleRTableEntity originalEntity = new SampleRTableEntity( jobType, string.Format(jobIdTemplate, i), string.Format(messageTemplate, i)); this.repTable.Execute(TableOperation.Insert(originalEntity)); partitionKey = originalEntity.PartitionKey; } // // Retrieve entities and use them to create batchOperation to Replace or Delete // IEnumerable<SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey); TableBatchOperation batchOperation = new TableBatchOperation(); int m = 0; foreach (SampleRTableEntity entity in allEntities) { Console.WriteLine("{0}", entity.ToString()); Console.WriteLine("---------------------------------------"); if (opTypes[m] == TableOperationType.Replace) { SampleRTableEntity replaceEntity = new SampleRTableEntity( entity.JobType, entity.JobId, string.Format(updatedMessageTemplate, m)) { ETag = entity._rtable_Version.ToString() }; batchOperation.Replace(replaceEntity); } else if (opTypes[m] == TableOperationType.Delete) { entity.ETag = entity._rtable_Version.ToString(); batchOperation.Delete(entity); } else { throw new ArgumentException( string.Format("opType={0} is NOT supported", opTypes[m]), "opType"); } m++; } // // Enable HttpMangler // Call this.repTable.ExecuteBatch(batchOperation) // this.RunHttpManglerBehaviorHelper( batchOperation, behaviors, targetApiExpectedToFail, out httpManglerStartTime); if (checkOriginalEntityUnchanged) { Console.WriteLine("Validate originalEntity remain unchanged."); allEntities = this.rtableWrapper.GetAllRows(partitionKey); batchOperation = new TableBatchOperation(); m = 0; foreach (SampleRTableEntity entity in allEntities) { Console.WriteLine("{0}", entity.ToString()); Console.WriteLine("---------------------------------------"); Assert.AreEqual(string.Format(jobType, m), entity.JobType, "JobType does not match"); Assert.AreEqual(string.Format(jobIdTemplate, m), entity.JobId, "JobId does not match"); Assert.AreEqual(string.Format(messageTemplate, m), entity.Message, "Message does not match"); m++; } Console.WriteLine("Passed validation"); } // // After httpMangler is turned off, read from individual accounts... // Console.WriteLine("\nAfter httpMangler is turned off, read from individual accounts..."); for (int i = 0; i < count; i++) { this.ReadFromIndividualAccountsDirectly( jobType, string.Format(jobIdTemplate, i), checkStorageAccountsConsistent); } return partitionKey; }
async Task <string> UploadHistoryBatch( string instanceId, string executionId, TableBatchOperation historyEventBatch, StringBuilder historyEventNamesBuffer, int numberOfTotalEvents, string eTagValue) { // Adding / updating sentinel entity DynamicTableEntity sentinelEntity = new DynamicTableEntity(instanceId, SentinelRowKey) { Properties = { ["ExecutionId"] = new EntityProperty(executionId), } }; if (!string.IsNullOrEmpty(eTagValue)) { sentinelEntity.ETag = eTagValue; historyEventBatch.Replace(sentinelEntity); } else { historyEventBatch.InsertOrReplace(sentinelEntity); } Stopwatch stopwatch = Stopwatch.StartNew(); IList <TableResult> tableResultList; try { tableResultList = await this.historyTable.ExecuteBatchAsync( historyEventBatch, this.StorageTableRequestOptions, null); } catch (StorageException ex) { if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed) { AnalyticsEventSource.Log.SplitBrainDetected( this.storageAccountName, this.taskHubName, instanceId, executionId, historyEventBatch.Count, numberOfTotalEvents, historyEventNamesBuffer.ToString(0, historyEventNamesBuffer.Length - 1), // remove trailing comma stopwatch.ElapsedMilliseconds, eTagValue); } throw; } this.stats.StorageRequests.Increment(); this.stats.TableEntitiesWritten.Increment(historyEventBatch.Count); if (tableResultList != null) { for (int i = tableResultList.Count - 1; i >= 0; i--) { if (((DynamicTableEntity)tableResultList[i].Result).RowKey == SentinelRowKey) { eTagValue = tableResultList[i].Etag; break; } } } AnalyticsEventSource.Log.AppendedInstanceHistory( this.storageAccountName, this.taskHubName, instanceId, executionId, historyEventBatch.Count, numberOfTotalEvents, historyEventNamesBuffer.ToString(0, historyEventNamesBuffer.Length - 1), // remove trailing comma stopwatch.ElapsedMilliseconds, this.GetETagValue(instanceId)); return(eTagValue); }
public static void Run([TimerTrigger("0 0 1 * * *")] TimerInfo myTimer, TraceWriter log) { log.Info($"Timer trigger function for CopyBlobReadStatsToTables executed at: {DateTime.Now}"); CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["AnalyticsStorage"]); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference("$logs"); CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); CloudTable table = tableClient.GetTableReference("BlobAnalytics"); if (container.Exists()) { foreach (CloudBlob analyticsBlob in container.ListBlobs(useFlatBlobListing:true)) { if (!analyticsBlob.Uri.ToString().Contains(DateTime.Now.ToString("yyyy/MM/dd"))) { var blobAnalyticsList = new List <BlobAnalytics>(); System.IO.Stream file = analyticsBlob.OpenRead(); var blobReader = new System.IO.StreamReader(file); using (var csvReader = new CsvReader(blobReader)) { csvReader.ReadHeaderRecord(); while (csvReader.HasMoreRecords) { csvReader.ValueSeparator = ';'; var record = csvReader.ReadDataRecord(); if (record[2].Trim() == "GetBlob" && record[7].Trim() == "anonymous" && record[20].Trim() != "0" && record[12].Contains(ConfigurationManager.AppSettings["PathToTrack"])) { var analytics = new BlobAnalytics() { RequestUrl = record[11], RequesterIpAddress = record[15], PartitionKey = record[12].Replace(ConfigurationManager.AppSettings["PathToTrack"], ""), UserAgentHeader = record[27], Referrer = record[28], RawData = record.ToString(), DownloadTime = record[1], RowKey = record[13] }; blobAnalyticsList.Add(analytics); var batchOperation = new TableBatchOperation(); batchOperation.Insert(analytics); TableOperation retrieveOperation = TableOperation.Retrieve <PartitionRowCount>(analytics.PartitionKey, "Count"); TableResult retrievedResult = table.Execute(retrieveOperation); var updateEntity = (PartitionRowCount)retrievedResult.Result; if (updateEntity == null) { updateEntity = new PartitionRowCount { PartitionKey = analytics.PartitionKey, RowKey = "Count" }; batchOperation.Insert(updateEntity); } else { updateEntity.Count++; batchOperation.Replace(updateEntity); } try { table.ExecuteBatch(batchOperation); } catch (StorageException ex) { if (ex.RequestInformation.HttpStatusCode != 409) { log.Info($"Exception for {analyticsBlob.Uri}: {ex}"); CloudBlobContainer failedLogsContainer = blobClient.GetContainerReference("failedlogs"); CloudBlockBlob targetBlob = failedLogsContainer.GetBlockBlobReference(analyticsBlob.Name); targetBlob.StartCopy((CloudBlockBlob)analyticsBlob); throw; } } } } } file.Close(); #if (DEBUG == false) analyticsBlob.Delete(); #endif } } } }