private static async Task AcceptSuggestionInternal([NotNull] AppEntity app, [NotNull] SuggestionEntity suggestion, int retries)
{
var batchOperation = new TableBatchOperation
{
TableOperation.Delete(suggestion),
TableOperation.InsertOrReplace(new ProcessedSuggestionEntity(suggestion))
};
if (suggestion.IsRetype)
{
batchOperation.Delete(app);
batchOperation.Insert(new AppEntity(app.SteamAppId, app.SteamName, suggestion.AppType));
}
else
{
batchOperation.Replace(app);
}
var table = await GetTable(SteamToHltbTableName, retries).ConfigureAwait(false);
CommonEventSource.Log.AcceptSuggestionStart(suggestion.SteamAppId, suggestion.HltbId);
await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false);
CommonEventSource.Log.AcceptSuggestionStop(suggestion.SteamAppId, suggestion.HltbId);
}
protected override TableBatchOperation PrepareNextBatchOperation(IList <T> batch, string partitionKey, out List <XTableResult> results)
{
// Search for existing entities versions
List <VersionedTableEntity> existingVersions = FindExistingVersions(batch, partitionKey);
results = new List <XTableResult>(batch.Count);
TableBatchOperation batchOperation = new TableBatchOperation();
// Add data operations
foreach (T entity in batch)
{
VersionedTableEntity existingEntity = existingVersions.FirstOrDefault(x => x.RowKey == entity.RowKey);
results.Add(new XTableResult());
// Insert entity if it doesn't exist
if (existingEntity == null)
{
batchOperation.Insert(entity);
}
else
{
entity.Version = VersionIncrementer.Increment(existingEntity.Version);
// Set current ETag
entity.ETag = existingEntity.ETag;
batchOperation.Replace(entity);
}
}
return(batchOperation);
}
private void AddOperationToBatch(ref TableBatchOperation tableBatchOperation, TAzureTableEntity entity, SaveType batchMethodName)
{
switch (batchMethodName)
{
case SaveType.Insert:
tableBatchOperation.Insert(entity);
break;
case SaveType.InsertOrMerge:
tableBatchOperation.InsertOrMerge(entity);
break;
case SaveType.InsertOrReplace:
tableBatchOperation.InsertOrReplace(entity);
break;
case SaveType.Merge:
tableBatchOperation.Merge(entity);
break;
case SaveType.Delete:
tableBatchOperation.Delete(entity);
break;
case SaveType.Replace:
tableBatchOperation.Replace(entity);
break;
}
}
private static void AddOperationToBatch(ref TableBatchOperation tableBatchOperation, TAzureTableEntity entity, string batchMethodName)
{
switch (batchMethodName)
{
case CtConstants.TableOpInsert:
tableBatchOperation.Insert(entity);
break;
case CtConstants.TableOpInsertOrMerge:
tableBatchOperation.InsertOrMerge(entity);
break;
case CtConstants.TableOpInsertOrReplace:
tableBatchOperation.InsertOrReplace(entity);
break;
case CtConstants.TableOpMerge:
tableBatchOperation.Merge(entity);
break;
case CtConstants.TableOpDelete:
entity.ETag = "*";
tableBatchOperation.Delete(entity);
break;
case CtConstants.TableOpReplace:
tableBatchOperation.Replace(entity);
break;
}
}
public static async Task UpdateAsync <TPayload>(
this CloudTable table, TPayload payload,
KeysPair keys)
where TPayload : class, new()
{
Ensure.ArgumentNotNull(keys, nameof(keys));
Ensure.ArgumentNotNull(payload, nameof(payload));
Ensure.ArgumentNotNull(table, nameof(table));
var entity = new DynamicTableEntity(keys.PartitionKey, keys.RowKey);
entity.Timestamp = DateTime.UtcNow;
entity.Properties = EntityPropertyConverter.Flatten(payload, new OperationContext());
entity.ETag = "*";
var batch = new TableBatchOperation();
batch.Replace(entity);
await table.ExecuteBatchAsync(batch)
.ConfigureAwait(false);
await table.ExecuteBatchAsync(batch)
.ConfigureAwait(false);
}
protected override async Task DeleteMessagesToAsync(string persistenceId, long toSequenceNr, bool isPermanent)
{
try
{
CloudTable table = _extension.TableJournalSettings
.GetClient(persistenceId)
.GetTableReference(_extension.TableJournalSettings.TableName);
IEnumerable <Event> results =
table.ExecuteQuery(BuildDeleteTableQuery(persistenceId, toSequenceNr))
.OrderByDescending(t => t.SequenceNr);
if (results.Any())
{
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (Event s in results)
{
s.IsDeleted = true;
if (isPermanent)
{
batchOperation.Delete(s);
}
else
{
batchOperation.Replace(s);
}
}
await table.ExecuteBatchAsync(batchOperation);
}
}
catch (Exception ex)
{
_log.Error(ex, ex.Message);
throw;
}
}
/// <summary>
/// Update collection of notes for individual personal goal in storage.
/// </summary>
/// <param name="personalGoalNoteEntities">Holds collection of personal goal notes data.</param>
/// <returns>A boolean that represents personal goal note detail entities are updated.</returns>
public async Task <bool> UpdatePersonalGoalNoteDetailsAsync(IEnumerable <PersonalGoalNoteDetail> personalGoalNoteEntities)
{
personalGoalNoteEntities = personalGoalNoteEntities ?? throw new ArgumentNullException(nameof(personalGoalNoteEntities));
try
{
await this.EnsureInitializedAsync();
TableBatchOperation tableBatchOperation = new TableBatchOperation();
int batchCount = (int)Math.Ceiling((double)personalGoalNoteEntities.Count() / PersonalGoalNotesPerBatch);
for (int batchCountIndex = 0; batchCountIndex < batchCount; batchCountIndex++)
{
var personalGoalNoteEntitiesBatch = personalGoalNoteEntities.Skip(batchCountIndex * PersonalGoalNotesPerBatch).Take(PersonalGoalNotesPerBatch);
foreach (var personalGoalNoteEntity in personalGoalNoteEntitiesBatch)
{
tableBatchOperation.Replace(personalGoalNoteEntity);
}
if (tableBatchOperation.Count > 0)
{
await this.CloudTable.ExecuteBatchAsync(tableBatchOperation);
}
}
return(true);
}
catch (Exception ex)
{
this.logger.LogError(ex, $"An error occurred in {nameof(this.CreateOrUpdatePersonalGoalNoteDetailsAsync)} while storing notes in storage.");
throw;
}
}
public async Task Add(TableOperation operation)
{
batchOperation.Add(operation);
if (operation.Entity == key)
{
batchContainsKey = true;
}
if (batchOperation.Count == AzureTableConstants.MaxBatchSize - (batchContainsKey ? 0 : 1))
{
// the key serves as a synchronizer, to prevent modification by multiple grains under edge conditions,
// like duplicate activations or deployments.Every batch write needs to include the key,
// even if the key values don't change.
if (!batchContainsKey)
{
if (string.IsNullOrEmpty(key.ETag))
{
batchOperation.Insert(key);
}
else
{
batchOperation.Replace(key);
}
}
await Flush().ConfigureAwait(false);
batchOperation.Clear();
batchContainsKey = false;
}
}
// TODO: add methods for
// delete single record
// Delete an entity
// Retrieve entities in pages asynchronously
/// <summary>
/// MS docs - https://docs.microsoft.com/en-us/azure/cosmos-db/table-storage-how-to-use-dotnet#retrieve-entities-in-pages-asynchronously
/// </summary>
// TODO: update that
//https://stackoverflow.com/questions/17955557/painfully-slow-azure-table-insert-and-delete-batch-operations
public void BatchUpdate <TEntity>(IList <TEntity> data, ILogger logger = null) where TEntity : ITableEntity
{
var groupedByPartition = data.GroupBy(x => x.PartitionKey).ToList();
foreach (var group in groupedByPartition)
{
var entities = group.Select(x => x).ToList();
logger?.Info("Saving for PartitionKey = " + group.Key + $" ({entities.Count}) records");
int rowOffset = 0;
while (rowOffset < entities.Count)
{
var rows = entities.Skip(rowOffset).Take(100).ToList();
rowOffset += rows.Count;
string partition = "$" + rowOffset.ToString();
var batch = new TableBatchOperation();
foreach (var row in rows)
{
batch.Replace(row);
}
CloudTable.ExecuteBatchAsync(batch);
logger?.Info("Updated batch for partition " + partition);
}
}
}
public void Replace(string partitionKey, string rowKey, string etag, string propertyName, object propertyValue)
{
Require.NotEmpty(partitionKey, "partitionKey");
Require.NotEmpty(rowKey, "rowKey");
Require.NotEmpty(etag, "etag");
Require.NotEmpty(propertyName, "propertyName");
AssertBatchSizeIsNotExceeded();
var entity = m_tableEntityConverter.CreateDynamicTableEntityFromProperties(propertyName, propertyValue);
entity.PartitionKey = partitionKey;
entity.RowKey = rowKey;
entity.ETag = etag;
m_batch.Replace(entity);
}
public void Replace(E entity)
{
CheckNotSaved();
var fat = ConvertToFatEntity(entity);
fat.ETag = fat.ETag ?? "*";
_context.Replace(fat);
_isDirty = true;
}
public void TableBatchReplaceSync()
{
// Insert Entity
Console.WriteLine("Calling Insert()...");
DynamicReplicatedTableEntity baseEntity = new DynamicReplicatedTableEntity("replace test", "foo");
baseEntity.Properties.Add("prop1", new EntityProperty("value1"));
this.repTable.Execute(TableOperation.Insert(baseEntity));
// ReplaceEntity
DynamicReplicatedTableEntity replaceEntity = new DynamicReplicatedTableEntity(baseEntity.PartitionKey, baseEntity.RowKey)
{
ETag = "1"
};
replaceEntity.Properties.Add("prop2", new EntityProperty("value2"));
Console.WriteLine("Calling Replace()...");
TableBatchOperation batch = new TableBatchOperation();
batch.Replace(replaceEntity);
this.repTable.ExecuteBatch(batch);
// Retrieve Entity & Verify Contents
Console.WriteLine("Calling Retrieve()...");
TableResult result = this.repTable.Execute(TableOperation.Retrieve <DynamicReplicatedTableEntity>(baseEntity.PartitionKey, baseEntity.RowKey));
DynamicReplicatedTableEntity retrievedEntity = result.Result as DynamicReplicatedTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]);
//
// Replace() again
//
Console.WriteLine("Calling Replace() again, setting Etag to {0}", retrievedEntity._rtable_Version);
replaceEntity = new DynamicReplicatedTableEntity(baseEntity.PartitionKey, baseEntity.RowKey)
{
ETag = retrievedEntity._rtable_Version.ToString()
};
replaceEntity.Properties.Add("prop3", new EntityProperty("value3"));
batch = new TableBatchOperation();
batch.Replace(replaceEntity);
this.repTable.ExecuteBatch(batch);
Console.WriteLine("Calling Retrieve()...");
result = this.repTable.Execute(TableOperation.Retrieve <DynamicReplicatedTableEntity>(baseEntity.PartitionKey, baseEntity.RowKey));
retrievedEntity = result.Result as DynamicReplicatedTableEntity;
Assert.IsNotNull(retrievedEntity);
Console.WriteLine("{0}", retrievedEntity.ToString());
Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["prop3"], retrievedEntity.Properties["prop3"]);
}
public void TableBatchAllSupportedOperationsSync()
{
TableBatchOperation batch = new TableBatchOperation();
string pk = Guid.NewGuid().ToString();
// insert
batch.Insert(GenerateRandomEnitity(pk));
// delete
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.Delete(entity);
}
// replace
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.Replace(entity);
}
// insert or replace
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.InsertOrReplace(entity);
}
// merge
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.Merge(entity);
}
// insert or merge
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.InsertOrMerge(entity);
}
IList <TableResult> results = this.repTable.ExecuteBatch(batch);
Assert.AreEqual(results.Count, 6);
IEnumerator <TableResult> enumerator = results.GetEnumerator();
for (int i = 0; i < results.Count; i++)
{
enumerator.MoveNext();
Assert.AreEqual((int)HttpStatusCode.NoContent, enumerator.Current.HttpStatusCode, "HttpStatusCode mismatch i={0}", i);
}
}
public IList <TableResult> UpdateEntities(string tableName, IList <StorageEntity> entities)
{
SetProperties(entities);
CloudTable table = _storage.GetTableReference(tableName);
TableBatchOperation operation = new TableBatchOperation();
foreach (StorageEntity entity in entities)
{
operation.Replace(entity);
}
return(table.ExecuteBatch(operation));
}
private static async Task ReplaceTableEntities(CloudTable table, IEnumerable <ITableEntity> entities)
{
await Task.WhenAll(
Group(entities, azureStorageBatchSize)
.Select(batch =>
{
TableBatchOperation opsBatch = new TableBatchOperation();
foreach (var item in batch)
{
opsBatch.Replace(item);
}
return(table.ExecuteBatchAsync(opsBatch));
}));
}
/// <summary>
/// Stores a user data element, overwriting the previous one if present.
/// </summary>
/// <param name="user">The user the data belongs to.</param>
/// <param name="key">The key of the data element (case insensitive).</param>
/// <param name="value">The value of the data element, <c>null</c> for deleting the data.</param>
/// <returns><c>true</c> if the data element is stored, <c>false</c> otherwise.</returns>
/// <exception cref="ArgumentNullException">If <paramref name="user"/> or <paramref name="key"/> are <c>null</c>.</exception>
/// <exception cref="ArgumentException">If <paramref name="key"/> is empty.</exception>
public bool StoreUserData(UserInfo user, string key, string value)
{
if (user == null)
{
throw new ArgumentNullException("user");
}
if (key == null)
{
throw new ArgumentNullException("key");
}
if (key.Length == 0)
{
throw new ArgumentException("Key cannot be empty", "key");
}
try {
// If the user does not exists return false
if (GetUserEntity(_wiki, user.Username) == null)
{
return(false);
}
CloudTable table = _cloudTableClient.GetTableReference(UserDataTable);
var batchOperation = new TableBatchOperation();
UserDataEntity userDataEntity = GetUserDataEntity(_wiki, user.Username, key);
if (userDataEntity == null)
{
userDataEntity = new UserDataEntity()
{
PartitionKey = _wiki + "|" + user.Username,
RowKey = key,
Value = value
};
batchOperation.Insert(userDataEntity);
}
else
{
userDataEntity.Value = value;
batchOperation.Replace(userDataEntity);
}
table.ExecuteBatch(batchOperation);
return(true);
}
catch (Exception ex) {
throw ex;
}
}
public virtual async Task <IEnumerable <TableResult> > UpdateRangeAndReturnTableResultsAsync(IEnumerable <TEntity> entities)
{
var batchOperation = new TableBatchOperation();
foreach (var entity in entities)
{
var tableEntityProxy = await _tableEntityProxyFactory.BuildAsync(entity);
batchOperation.Replace(tableEntityProxy);
}
var results = await ExecuteBatchAsLimitedBatchesAsync(batchOperation);
return(results);
}
public static IEnumerable <TableBatchInformation <T> > ReplaceAll <T>(this IEnumerable <IEnumerable <T> > batches)
where T : ITableEntity
{
foreach (var batch in batches.Select(b => b.ToArray()))
{
var operation = new TableBatchOperation();
foreach (var instance in batch)
{
operation.Replace(instance);
}
yield return(new TableBatchInformation <T>(operation, batch, TableOperationType.Replace));
}
}
public static async Task <IActionResult> MarkAsDone(
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = null)]
HttpRequest req,
[Table("todo")] CloudTable todoTable,
ILogger log)
{
if (!req.Query.ContainsKey("id"))
{
return(new BadRequestResult());
}
string uid = req.Query["id"];
Guid uuid;
if (!Guid.TryParse(uid, out uuid))
{
return(new NotFoundResult());
}
try
{
var query = new TableQuery <ToDoItem>().Where(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, uid)
);
var result = await todoTable.ExecuteQuerySegmentedAsync(query, null);
var elem = result.First();
elem.Done = true;
elem.DoneTimestamp = (long)DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1)).TotalSeconds;
var batch = new TableBatchOperation();
batch.Replace(elem);
await todoTable.ExecuteBatchAsync(batch);
return(new OkObjectResult(elem.ToToDoDTO()));
}
catch (Exception e)
{
// Return 404 if process fails since most of the cases fail due to invalid IDs
log.LogCritical(e, "Delete Failed");
return(new NotFoundResult());
}
}
public async Task <string> Persist(string expectedETag, string metadata, List <PendingTransactionState <TState> > statesToPrepare)
{
try
{
var batchOperation = new TableBatchOperation();
this.key.ETag = expectedETag;
this.key.Metadata = metadata;
if (string.IsNullOrEmpty(this.key.ETag))
{
batchOperation.Insert(this.key);
}
else
{
batchOperation.Replace(this.key);
}
// add new states
List <Tuple <string, long> > stored = this.states.Select(s => Tuple.Create(s.TransactionId, s.SequenceId)).ToList();
List <StateEntity> newStates = new List <StateEntity>();
foreach (PendingTransactionState <TState> pendingState in statesToPrepare.Where(p => !stored.Contains(Tuple.Create(p.TransactionId, p.SequenceId))))
{
var newState = StateEntity.Create(this.jsonSettings, this.partition, pendingState);
newStates.Add(newState);
batchOperation.Insert(newState);
}
if (batchOperation.Count > AzureTableConstants.MaxBatchSize)
{
this.logger.LogError("Too many pending states. PendingStateCount {PendingStateCount}.", batchOperation.Count);
throw new InvalidOperationException($"Too many pending states. PendingStateCount {batchOperation.Count}");
}
await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false);
this.states.AddRange(newStates);
return(this.key.ETag);
}
catch (Exception ex)
{
this.logger.LogError("Transactional state persist failed {Exception}.", ex);
throw;
}
}
/// <summary>
/// Stores the value of a Setting.
/// </summary>
/// <param name="name">The name of the Setting.</param>
/// <param name="value">The value of the Setting. Value cannot contain CR and LF characters, which will be removed.</param>
/// <returns>True if the Setting is stored, false otherwise.</returns>
/// <exception cref="ArgumentNullException">If <paramref name="name"/> is <c>null</c>.</exception>
/// <exception cref="ArgumentException">If <paramref name="name"/> is empty.</exception>
public bool SetSetting(string name, string value)
{
if (name == null)
{
throw new ArgumentNullException("name");
}
if (name.Length == 0)
{
throw new ArgumentException("name");
}
_settingsDictionary = null;
// Nulls are converted to empty strings
if (value == null)
{
value = "";
}
GlobalSettingsEntity settingsEntity = GetGlobalSettingsEntity(name);
CloudTable table = _cloudTableClient.GetTableReference(GlobalSettingsTable);
var batchOperation = new TableBatchOperation();
if (settingsEntity == null)
{
settingsEntity = new GlobalSettingsEntity()
{
PartitionKey = "0",
RowKey = name,
Value = value
};
batchOperation.Insert(settingsEntity);
}
else
{
settingsEntity.Value = value;
batchOperation.Replace(settingsEntity);
}
table.ExecuteBatch(batchOperation);
return(true);
}
private static async Task DeleteSuggestionInternal([NotNull] SuggestionEntity suggestion, AppEntity app, int retries)
{
var batchOperation = new TableBatchOperation
{
TableOperation.Delete(suggestion),
TableOperation.InsertOrReplace(new ProcessedSuggestionEntity(suggestion))
};
if (app != null) //this means we want to update the app to be a verified game and stop future non-game suggestions
{
app.VerifiedGame = true;
batchOperation.Replace(app);
}
var table = await GetTable(SteamToHltbTableName, retries).ConfigureAwait(false);
CommonEventSource.Log.DeleteSuggestionStart(suggestion.SteamAppId, suggestion.HltbId);
await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false);
CommonEventSource.Log.DeleteSuggestionStop(suggestion.SteamAppId, suggestion.HltbId);
}
public async Task RunOrchestrator(
[OrchestrationTrigger] IDurableOrchestrationContext context,
[Table("population")] CloudTable storage,
[Table("generation")] CloudTable generation,
[Queue("population-roulette")] ICollector <string> populationRulet,
ILogger log)
{
log.LogInformation($"Population Encounter orchestrator started");
var population = context.GetInput <IList <Population> >();
var taskList = from pop in population
select context.CallActivityAsync <Population>("PupulationEncounter_FitnessFunction", pop);
log.LogInformation($"Run tasks for Fitness calculation");
var encountedpopulation = new List <Population>(await Task.WhenAll(taskList.ToArray()));
var populationFitnessSum = encountedpopulation.Sum(a => a.Fitness);
var batchUpdateOperation = new TableBatchOperation();
log.LogInformation($"Run tasks for Adaptation calculation");
foreach (var pop in encountedpopulation)
{
pop.Adaptation = (pop.Fitness / populationFitnessSum);
batchUpdateOperation.Replace(pop);
}
log.LogInformation($"Save updated data about population to DB");
storage.ExecuteBatch(batchUpdateOperation);
var generationNumber = encountedpopulation.FirstOrDefault().PartitionKey;
TableOperation insertOrMergeOperation = TableOperation.InsertOrMerge(new Generation()
{
PartitionKey = generationNumber,
RowKey = generationNumber,
PopulationFitness = populationFitnessSum,
BestResult = encountedpopulation.Where(a => a.Adaptation == encountedpopulation.Max(a => a.Adaptation)).Select(a => a.Value.ToString()).Aggregate((a, b) => a + ", " + b)
});
generation.Execute(insertOrMergeOperation);
log.LogInformation("Add message to rulette queue.");
populationRulet.Add(generationNumber);
}
/// <summary>
/// Updates the entity batch async.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entities">The entities.</param>
/// <param name="entitiesToAdd">The entities to add.</param>
/// <param name="entitiesToDelete">The entities to delete.</param>
/// <param name="replace">if set to <c>true</c> [force].</param>
/// <returns>Task{OperationResult}.</returns>
public async Task <OperationResult> UpdateEntityBatchAsync <T>(
IList <T> entities,
IList <T> entitiesToAdd = null,
IList <T> entitiesToDelete = null,
bool replace = true) where T : TableEntity
{
var batchOperation = new TableBatchOperation();
foreach (var entity in entities)
{
if (replace)
{
batchOperation.Replace(entity);
}
else
{
batchOperation.Merge(entity);
}
}
return(await CompleteUpdateDeleteBatchWithRestOperationsAndStart(batchOperation, entitiesToAdd, entitiesToDelete));
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] ChangePasswordInfo info,
ILogger log)
{
var table = TableUtilities.GetCloudTable("Applications");
var operation = TableOperation.Retrieve <ApplicationEntity>(info.GroupId, info.ApplicationId);
var result = await table.ExecuteAsync(operation);
var application = result.Result as ApplicationEntity;
if (application == null)
{
return(new NotFoundResult());
}
if (application.Password.Equals(info.Password) == false)
{
return(new UnauthorizedResult());
}
var query = table.CreateQuery <ApplicationEntity>()
.Where(p => p.PartitionKey.Equals(info.GroupId))
.AsTableQuery();
var entities = await table.ExecuteQueryAsync(query);
var batch = new TableBatchOperation();
foreach (var entity in entities)
{
entity.Password = info.NewPassword;
batch.Replace(entity);
}
await table.ExecuteBatchAsync(batch);
return(new NoContentResult());
}
protected override TableBatchOperation PrepareNextBatchOperation(IList <DynamicTableEntity> batch, string partitionKey, out List <XTableResult> results)
{
// Search for existing entities versions
List <VersionedTableEntity> existingVersions = FindExistingVersions(batch, partitionKey);
results = new List <XTableResult>(batch.Count);
TableBatchOperation batchOperation = new TableBatchOperation();
// Add data operations
for (int i = 0; i < batch.Count; ++i)
{
VersionedTableEntity existingEntity = existingVersions.FirstOrDefault(x => x.RowKey == batch[i].RowKey);
results.Add(new XTableResult());
// Insert entry if it doesn't exist
if (existingEntity == null)
{
batchOperation.Insert(batch[i]);
}
else
{
// Replace entity only if its version is higher than existing one
if (batch[i].Properties.ContainsKey(VersionColumn) && batch[i].Properties[VersionColumn].Int64Value > existingEntity.Version)
{
// Set current ETag
batch[i].ETag = existingEntity.ETag;
batchOperation.Replace(batch[i]);
}
else
{
results[i].Discarded = true;
}
}
}
return(batchOperation);
}
public static void Execute <T>(this TableBatchOperation operation, TableOperationType type, T entity)
where T : ITableEntity
{
switch (type)
{
case TableOperationType.Insert:
operation.Insert(entity);
break;
case TableOperationType.Delete:
operation.Delete(entity);
break;
case TableOperationType.Replace:
operation.Replace(entity);
break;
case TableOperationType.Merge:
operation.Merge(entity);
break;
case TableOperationType.InsertOrReplace:
operation.InsertOrReplace(entity);
break;
case TableOperationType.InsertOrMerge:
operation.InsertOrMerge(entity);
break;
case TableOperationType.Retrieve:
operation.Retrieve(entity.PartitionKey, entity.RowKey);
break;
default:
throw new Exception($"Invalid enum value {nameof(TableOperationType)} {type}");
}
}
public void TableBatchAllSupportedOperationsSync()
{
CloudTableClient tableClient = GenerateCloudTableClient();
TableBatchOperation batch = new TableBatchOperation();
string pk = Guid.NewGuid().ToString();
// insert
batch.Insert(GenerateRandomEnitity(pk));
// delete
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.Delete(entity);
}
// replace
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.Replace(entity);
}
// insert or replace
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.InsertOrReplace(entity);
}
// merge
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.Merge(entity);
}
// insert or merge
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.InsertOrMerge(entity);
}
IList<TableResult> results = currentTable.ExecuteBatch(batch);
Assert.AreEqual(results.Count, 6);
IEnumerator<TableResult> enumerator = results.GetEnumerator();
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.Created);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
}
public void TableBatchReplaceFailAPM()
{
CloudTableClient tableClient = GenerateCloudTableClient();
// Insert Entity
DynamicTableEntity baseEntity = new DynamicTableEntity("merge test", "foo");
baseEntity.Properties.Add("prop1", new EntityProperty("value1"));
currentTable.Execute(TableOperation.Insert(baseEntity));
string staleEtag = baseEntity.ETag;
// update entity to rev etag
baseEntity.Properties["prop1"].StringValue = "updated value";
currentTable.Execute(TableOperation.Replace(baseEntity));
OperationContext opContext = new OperationContext();
try
{
// Attempt a merge with stale etag
DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = staleEtag };
replaceEntity.Properties.Add("prop2", new EntityProperty("value2"));
TableBatchOperation batch = new TableBatchOperation();
batch.Replace(replaceEntity);
using (ManualResetEvent evt = new ManualResetEvent(false))
{
IAsyncResult asyncRes = null;
currentTable.BeginExecuteBatch(batch, null, opContext, (res) =>
{
asyncRes = res;
evt.Set();
}, null);
evt.WaitOne();
currentTable.EndExecuteBatch(asyncRes);
}
Assert.Fail();
}
catch (StorageException)
{
TestHelper.ValidateResponse(opContext,
1,
(int)HttpStatusCode.PreconditionFailed,
new string[] { "UpdateConditionNotSatisfied", "ConditionNotMet" },
new string[] { "The update condition specified in the request was not satisfied.", "The condition specified using HTTP conditional header(s) is not met." });
}
// Delete Entity
currentTable.Execute(TableOperation.Delete(baseEntity));
opContext = new OperationContext();
// try replacing with deleted entity
try
{
DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = baseEntity.ETag };
replaceEntity.Properties.Add("prop2", new EntityProperty("value2"));
TableBatchOperation batch = new TableBatchOperation();
batch.Replace(replaceEntity);
using (ManualResetEvent evt = new ManualResetEvent(false))
{
IAsyncResult asyncRes = null;
currentTable.BeginExecuteBatch(batch, null, opContext, (res) =>
{
asyncRes = res;
evt.Set();
}, null);
evt.WaitOne();
currentTable.EndExecuteBatch(asyncRes);
}
Assert.Fail();
}
catch (StorageException)
{
TestHelper.ValidateResponse(opContext, 1, (int)HttpStatusCode.NotFound, new string[] { "ResourceNotFound" }, "The specified resource does not exist.");
}
}
public void TableBatchReplaceAPM()
{
CloudTableClient tableClient = GenerateCloudTableClient();
// Insert Entity
DynamicTableEntity baseEntity = new DynamicTableEntity("merge test", "foo");
baseEntity.Properties.Add("prop1", new EntityProperty("value1"));
currentTable.Execute(TableOperation.Insert(baseEntity));
// ReplaceEntity
DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = baseEntity.ETag };
replaceEntity.Properties.Add("prop2", new EntityProperty("value2"));
TableBatchOperation batch = new TableBatchOperation();
batch.Replace(replaceEntity);
using (ManualResetEvent evt = new ManualResetEvent(false))
{
IAsyncResult asyncRes = null;
currentTable.BeginExecuteBatch(batch, (res) =>
{
asyncRes = res;
evt.Set();
}, null);
evt.WaitOne();
currentTable.EndExecuteBatch(asyncRes);
}
// Retrieve Entity & Verify Contents
TableResult result = currentTable.Execute(TableOperation.Retrieve(baseEntity.PartitionKey, baseEntity.RowKey));
DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]);
}
public void Execute(TableBatchOperation batchOperation, TableEntity entity)
{
entity.ETag = "*"; // Always overwrite (ignore concurrency).
batchOperation.Replace(entity);
}
public void TableBatchAllSupportedOperationsSync()
{
TableBatchOperation batch = new TableBatchOperation();
string pk = Guid.NewGuid().ToString();
// insert
batch.Insert(GenerateRandomEnitity(pk));
// delete
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.Delete(entity);
}
// replace
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.Replace(entity);
}
// insert or replace
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.InsertOrReplace(entity);
}
// merge
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.Merge(entity);
}
// insert or merge
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.InsertOrMerge(entity);
}
IList<TableResult> results = this.repTable.ExecuteBatch(batch);
Assert.AreEqual(results.Count, 6);
IEnumerator<TableResult> enumerator = results.GetEnumerator();
for (int i = 0; i < results.Count; i++)
{
enumerator.MoveNext();
Assert.AreEqual((int)HttpStatusCode.NoContent, enumerator.Current.HttpStatusCode, "HttpStatusCode mismatch i={0}", i);
}
}
private async Task DoTableBatchAllSupportedOperationsAsync(TablePayloadFormat format)
{
tableClient.DefaultRequestOptions.PayloadFormat = format;
TableBatchOperation batch = new TableBatchOperation();
string pk = Guid.NewGuid().ToString();
// insert
batch.Insert(GenerateRandomEntity(pk));
// delete
{
DynamicTableEntity entity = GenerateRandomEntity(pk);
await currentTable.ExecuteAsync(TableOperation.Insert(entity));
batch.Delete(entity);
}
// replace
{
DynamicTableEntity entity = GenerateRandomEntity(pk);
await currentTable.ExecuteAsync(TableOperation.Insert(entity));
batch.Replace(entity);
}
// insert or replace
{
DynamicTableEntity entity = GenerateRandomEntity(pk);
await currentTable.ExecuteAsync(TableOperation.Insert(entity));
batch.InsertOrReplace(entity);
}
// merge
{
DynamicTableEntity entity = GenerateRandomEntity(pk);
await currentTable.ExecuteAsync(TableOperation.Insert(entity));
batch.Merge(entity);
}
// insert or merge
{
DynamicTableEntity entity = GenerateRandomEntity(pk);
await currentTable.ExecuteAsync(TableOperation.Insert(entity));
batch.InsertOrMerge(entity);
}
IList<TableResult> results = await currentTable.ExecuteBatchAsync(batch);
Assert.AreEqual(results.Count, 6);
IEnumerator<TableResult> enumerator = results.GetEnumerator();
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.Created);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
}
public void TableBatchReplaceSync()
{
CloudTableClient tableClient = GenerateCloudTableClient();
// Insert Entity
DynamicTableEntity baseEntity = new DynamicTableEntity("merge test", "foo");
baseEntity.Properties.Add("prop1", new EntityProperty("value1"));
currentTable.Execute(TableOperation.Insert(baseEntity));
// ReplaceEntity
DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = baseEntity.ETag };
replaceEntity.Properties.Add("prop2", new EntityProperty("value2"));
TableBatchOperation batch = new TableBatchOperation();
batch.Replace(replaceEntity);
currentTable.ExecuteBatch(batch);
// Retrieve Entity & Verify Contents
TableResult result = currentTable.Execute(TableOperation.Retrieve(baseEntity.PartitionKey, baseEntity.RowKey));
DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]);
}
async Task DoRandomAtomicCalls()
{
for (int callNum = 0; callNum < MigrationModel.NUM_CALLS_PER_MACHINE; callNum++)
{
TableCall originalCall;
MirrorTableCall referenceCall;
SortedDictionary <PrimaryKey, DynamicTableEntity> dump = await peekProxy.DumpReferenceTableAsync();
if (PSharpRuntime.Nondeterministic())
{
// Query
// XXX: Test the filtering?
var query = new TableQuery <DynamicTableEntity>();
query.FilterString = TableQuery.GenerateFilterCondition(
TableConstants.PartitionKey, QueryComparisons.Equal, MigrationModel.SINGLE_PARTITION_KEY);
// async/await pair needed to upcast the return value to object.
originalCall = async table => await table.ExecuteQueryAtomicAsync(query);
referenceCall = async referenceTable => await referenceTable.ExecuteQueryAtomicAsync(query);
Console.WriteLine("{0} starting query", machineId);
}
else
{
// Batch write
int batchSize = PSharpRuntime.Nondeterministic() ? 2 : 1;
var batch = new TableBatchOperation();
var rowKeyChoices = new List <string> {
"0", "1", "2", "3", "4", "5"
};
for (int opNum = 0; opNum < batchSize; opNum++)
{
int opTypeNum = PSharpNondeterminism.Choice(7);
int rowKeyI = PSharpNondeterminism.Choice(rowKeyChoices.Count);
string rowKey = rowKeyChoices[rowKeyI];
rowKeyChoices.RemoveAt(rowKeyI); // Avoid duplicate in same batch
var primaryKey = new PrimaryKey(MigrationModel.SINGLE_PARTITION_KEY, rowKey);
string eTag = null;
if (opTypeNum >= 1 && opTypeNum <= 3)
{
DynamicTableEntity existingEntity;
int etagTypeNum = PSharpNondeterminism.Choice(
dump.TryGetValue(primaryKey, out existingEntity) ? 3 : 2);
switch (etagTypeNum)
{
case 0: eTag = ChainTable2Constants.ETAG_ANY; break;
case 1: eTag = "wrong"; break;
case 2: eTag = existingEntity.ETag; break;
}
}
DynamicTableEntity entity = new DynamicTableEntity
{
PartitionKey = MigrationModel.SINGLE_PARTITION_KEY,
RowKey = rowKey,
ETag = eTag,
Properties = new Dictionary <string, EntityProperty> {
// Give us something to see on merge. Might help with tracing too!
{ string.Format("{0}_c{1}_o{2}", machineId.ToString(), callNum, opNum),
new EntityProperty(true) }
}
};
switch (opTypeNum)
{
case 0: batch.Insert(entity); break;
case 1: batch.Replace(entity); break;
case 2: batch.Merge(entity); break;
case 3: batch.Delete(entity); break;
case 4: batch.InsertOrReplace(entity); break;
case 5: batch.InsertOrMerge(entity); break;
case 6:
entity.ETag = ChainTable2Constants.ETAG_DELETE_IF_EXISTS;
batch.Delete(entity); break;
}
}
TableBatchOperation batchCopy = ChainTableUtils.CopyBatch <DynamicTableEntity>(batch);
originalCall = async table => await table.ExecuteBatchAsync(batch);
referenceCall = async referenceTable => await referenceTable.ExecuteMirrorBatchAsync(batchCopy, successfulBatchResult);
Console.WriteLine("{0} starting batch {1}", machineId, batch);
}
await RunCallAsync(originalCall, referenceCall);
Console.WriteLine("{0} table call verified");
}
}
private async Task DoTableBatchReplaceAsync(TablePayloadFormat format)
{
tableClient.DefaultRequestOptions.PayloadFormat = format;
// Insert Entity
DynamicTableEntity baseEntity = new DynamicTableEntity("merge test", "foo" + format.ToString());
baseEntity.Properties.Add("prop1", new EntityProperty("value1"));
await currentTable.ExecuteAsync(TableOperation.Insert(baseEntity));
// ReplaceEntity
DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = baseEntity.ETag };
replaceEntity.Properties.Add("prop2", new EntityProperty("value2"));
TableBatchOperation batch = new TableBatchOperation();
batch.Replace(replaceEntity);
await currentTable.ExecuteBatchAsync(batch);
// Retrieve Entity & Verify Contents
TableResult result = await currentTable.ExecuteAsync(TableOperation.Retrieve(baseEntity.PartitionKey, baseEntity.RowKey));
DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]);
}
private void DoEscapeTest(string data, bool useBatch, bool includeKey)
{
DynamicTableEntity ent = new DynamicTableEntity(includeKey ? "temp" + data : "temp", Guid.NewGuid().ToString());
ent.Properties.Add("foo", new EntityProperty(data));
// Insert
if (useBatch)
{
TableBatchOperation batch = new TableBatchOperation();
batch.Insert(ent);
currentTable.ExecuteBatch(batch);
}
else
{
currentTable.Execute(TableOperation.Insert(ent));
}
// Retrieve
TableResult res = null;
if (useBatch)
{
TableBatchOperation batch = new TableBatchOperation();
batch.Retrieve(ent.PartitionKey, ent.RowKey);
res = (currentTable.ExecuteBatch(batch))[0];
}
else
{
res = currentTable.Execute(TableOperation.Retrieve(ent.PartitionKey, ent.RowKey));
}
// Check equality
DynamicTableEntity retrievedEntity = res.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey);
Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey);
Assert.AreEqual(ent.ETag, retrievedEntity.ETag);
Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]);
// Query using data filter
TableQuery query = new TableQuery();
query.Where(string.Format(
"(PartitionKey eq \'{0}\') and (RowKey eq \'{1}\') and (foo eq \'{2}\')",
ent.PartitionKey,
ent.RowKey,
data.Replace("\'", "\'\'")));
retrievedEntity = currentTable.ExecuteQuery(query).Single();
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey);
Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey);
Assert.AreEqual(ent.ETag, retrievedEntity.ETag);
Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]);
// Merge
ent.Properties.Add("foo2", new EntityProperty("bar2"));
if (useBatch)
{
TableBatchOperation batch = new TableBatchOperation();
batch.Merge(ent);
currentTable.ExecuteBatch(batch);
}
else
{
currentTable.Execute(TableOperation.Merge(ent));
}
// Retrieve
if (useBatch)
{
TableBatchOperation batch = new TableBatchOperation();
batch.Retrieve(ent.PartitionKey, ent.RowKey);
res = (currentTable.ExecuteBatch(batch))[0];
}
else
{
res = currentTable.Execute(TableOperation.Retrieve(ent.PartitionKey, ent.RowKey));
}
retrievedEntity = res.Result as DynamicTableEntity;
Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey);
Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey);
Assert.AreEqual(ent.ETag, retrievedEntity.ETag);
Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]);
// Replace
ent.Properties.Remove("foo2");
ent.Properties.Add("foo3", new EntityProperty("bar3"));
if (useBatch)
{
TableBatchOperation batch = new TableBatchOperation();
batch.Replace(ent);
currentTable.ExecuteBatch(batch);
}
else
{
currentTable.Execute(TableOperation.Replace(ent));
}
// Retrieve
if (useBatch)
{
TableBatchOperation batch = new TableBatchOperation();
batch.Retrieve(ent.PartitionKey, ent.RowKey);
res = (currentTable.ExecuteBatch(batch))[0];
}
else
{
res = currentTable.Execute(TableOperation.Retrieve(ent.PartitionKey, ent.RowKey));
}
retrievedEntity = res.Result as DynamicTableEntity;
Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey);
Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey);
Assert.AreEqual(ent.ETag, retrievedEntity.ETag);
Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]);
}
async Task DoRandomAtomicCalls()
{
for (int callNum = 0; callNum < MigrationModel.NUM_CALLS_PER_MACHINE; callNum++)
{
SortedDictionary<PrimaryKey, DynamicTableEntity> dump = await peekProxy.DumpReferenceTableAsync();
if (PSharpRuntime.Nondeterministic())
{
// Query
var query = new TableQuery<DynamicTableEntity>();
query.FilterString = ChainTableUtils.CombineFilters(
TableQuery.GenerateFilterCondition(
TableConstants.PartitionKey, QueryComparisons.Equal, MigrationModel.SINGLE_PARTITION_KEY),
TableOperators.And,
NondeterministicUserPropertyFilterString());
await RunQueryAtomicAsync(query);
}
else
{
// Batch write
int batchSize = PSharpRuntime.Nondeterministic() ? 2 : 1;
var batch = new TableBatchOperation();
var rowKeyChoices = new List<string> { "0", "1", "2", "3", "4", "5" };
for (int opNum = 0; opNum < batchSize; opNum++)
{
int opTypeNum = PSharpNondeterminism.Choice(7);
int rowKeyI = PSharpNondeterminism.Choice(rowKeyChoices.Count);
string rowKey = rowKeyChoices[rowKeyI];
rowKeyChoices.RemoveAt(rowKeyI); // Avoid duplicate in same batch
var primaryKey = new PrimaryKey(MigrationModel.SINGLE_PARTITION_KEY, rowKey);
string eTag = null;
if (opTypeNum >= 1 && opTypeNum <= 3)
{
DynamicTableEntity existingEntity;
int etagTypeNum = PSharpNondeterminism.Choice(
dump.TryGetValue(primaryKey, out existingEntity) ? 3 : 2);
switch (etagTypeNum)
{
case 0: eTag = ChainTable2Constants.ETAG_ANY; break;
case 1: eTag = "wrong"; break;
case 2: eTag = existingEntity.ETag; break;
}
}
DynamicTableEntity entity = new DynamicTableEntity
{
PartitionKey = MigrationModel.SINGLE_PARTITION_KEY,
RowKey = rowKey,
ETag = eTag,
Properties = new Dictionary<string, EntityProperty> {
// Give us something to see on merge. Might help with tracing too!
{ string.Format("{0}_c{1}_o{2}", machineId.ToString(), callNum, opNum),
new EntityProperty(true) },
// Property with 50%/50% distribution for use in filters.
{ "isHappy", new EntityProperty(PSharpRuntime.Nondeterministic()) }
}
};
switch (opTypeNum)
{
case 0: batch.Insert(entity); break;
case 1: batch.Replace(entity); break;
case 2: batch.Merge(entity); break;
case 3: batch.Delete(entity); break;
case 4: batch.InsertOrReplace(entity); break;
case 5: batch.InsertOrMerge(entity); break;
case 6:
entity.ETag = ChainTable2Constants.ETAG_DELETE_IF_EXISTS;
batch.Delete(entity); break;
}
}
await RunBatchAsync(batch);
}
}
}
public void TableBatchReplaceSync()
{
// Insert Entity
Console.WriteLine("Calling Insert()...");
DynamicReplicatedTableEntity baseEntity = new DynamicReplicatedTableEntity("replace test", "foo");
baseEntity.Properties.Add("prop1", new EntityProperty("value1"));
this.repTable.Execute(TableOperation.Insert(baseEntity));
// ReplaceEntity
DynamicReplicatedTableEntity replaceEntity = new DynamicReplicatedTableEntity(baseEntity.PartitionKey, baseEntity.RowKey)
{
ETag = "1"
};
replaceEntity.Properties.Add("prop2", new EntityProperty("value2"));
Console.WriteLine("Calling Replace()...");
TableBatchOperation batch = new TableBatchOperation();
batch.Replace(replaceEntity);
this.repTable.ExecuteBatch(batch);
// Retrieve Entity & Verify Contents
Console.WriteLine("Calling Retrieve()...");
TableResult result = this.repTable.Execute(TableOperation.Retrieve<DynamicReplicatedTableEntity>(baseEntity.PartitionKey, baseEntity.RowKey));
DynamicReplicatedTableEntity retrievedEntity = result.Result as DynamicReplicatedTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]);
//
// Replace() again
//
Console.WriteLine("Calling Replace() again, setting Etag to {0}", retrievedEntity._rtable_Version);
replaceEntity = new DynamicReplicatedTableEntity(baseEntity.PartitionKey, baseEntity.RowKey)
{
ETag = retrievedEntity._rtable_Version.ToString()
};
replaceEntity.Properties.Add("prop3", new EntityProperty("value3"));
batch = new TableBatchOperation();
batch.Replace(replaceEntity);
this.repTable.ExecuteBatch(batch);
Console.WriteLine("Calling Retrieve()...");
result = this.repTable.Execute(TableOperation.Retrieve<DynamicReplicatedTableEntity>(baseEntity.PartitionKey, baseEntity.RowKey));
retrievedEntity = result.Result as DynamicReplicatedTableEntity;
Assert.IsNotNull(retrievedEntity);
Console.WriteLine("{0}", retrievedEntity.ToString());
Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["prop3"], retrievedEntity.Properties["prop3"]);
}
public void TableBatchAllSupportedOperationsAPM()
{
CloudTableClient tableClient = GenerateCloudTableClient();
TableBatchOperation batch = new TableBatchOperation();
string pk = Guid.NewGuid().ToString();
// insert
batch.Insert(GenerateRandomEnitity(pk));
// delete
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.Delete(entity);
}
// replace
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.Replace(entity);
}
// insert or replace
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.InsertOrReplace(entity);
}
// merge
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.Merge(entity);
}
// insert or merge
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.InsertOrMerge(entity);
}
IList<TableResult> results = null;
using (ManualResetEvent evt = new ManualResetEvent(false))
{
IAsyncResult asyncRes = null;
currentTable.BeginExecuteBatch(batch, (res) =>
{
asyncRes = res;
evt.Set();
}, null);
evt.WaitOne();
results = currentTable.EndExecuteBatch(asyncRes);
}
Assert.AreEqual(results.Count, 6);
IEnumerator<TableResult> enumerator = results.GetEnumerator();
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.Created);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
}
/// <summary>
/// Helper function to execute the specified BatchOperation after HttpMangler is turned off and validate correctness.
/// </summary>
/// <param name="count">Number of operations in the batch</param>
/// <param name="partitionKey">partitionKey to operate on</param>
/// <param name="jobType">partitionKey is generated from jobType</param>
/// <param name="jobId">RowKey is generated from jobId. JobIdTemplate = jobId-{0}</param>
/// <param name="opTypes">Specifies the batch operation to be performed</param>
protected void ExecuteBatchOperationAndValidate(
int count,
string partitionKey,
string jobType,
string jobId,
List<TableOperationType> opTypes)
{
Console.WriteLine("\nExecuteBatchOperationAndValidate(): Trying to batch update {0} entities...", count);
Assert.IsNotNull(opTypes, "opTypes = null");
Assert.AreEqual(count, opTypes.Count, "count and opTypes.Count should be the same");
string jobIdTemplate = jobId + "-{0}";
string replaceMessageTemplate = "updated-after-httpMangler-{0}";
IEnumerable<SampleRTableEntity> allEntities = null;
TableBatchOperation batchOperation = null;
int m = 0;
bool gotExceptionInLastAttempt = true;
int retries = 0;
while (retries < MaxRetries)
{
try
{
//
// GetAllRows()
//
allEntities = this.rtableWrapper.GetAllRows(partitionKey);
//
// Create a batchOperation to perform the specified opTypes
//
batchOperation = new TableBatchOperation();
m = 0;
foreach (SampleRTableEntity entity in allEntities)
{
if (opTypes[m] == TableOperationType.Replace)
{
// set up the new entity to be used in the batch operation
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(replaceMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
// add the operation to the batch operation
batchOperation.Replace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.Delete)
{
batchOperation.Delete(entity);
}
else
{
throw new ArgumentException(
string.Format("opType={0} is NOT supported", opTypes[m]),
"opType");
}
m++;
}
//
// Call this.repTable.ExecuteBatch(batchOperation);
//
if (batchOperation.Count == 0)
{
Console.WriteLine("retries={0}. Done. batchOperation.Count == 0", retries);
}
else
{
this.repTable.ExecuteBatch(batchOperation);
Console.WriteLine("retries={0}. Done ExecuteBatch()", retries);
}
gotExceptionInLastAttempt = false;
break;
}
catch (RTableConflictException ex)
{
Console.WriteLine("retries={0}. ExecuteBatch() got an RTableConflictException: {1}",
retries, ex.ToString());
retries++;
gotExceptionInLastAttempt = true;
Thread.Sleep(ConflictExceptionSleepTimeInMsec);
}
}
Console.WriteLine("gotExceptionInLastAttempt={0} retries={1} MaxRetries={2}",
gotExceptionInLastAttempt, retries, MaxRetries);
Assert.IsFalse(gotExceptionInLastAttempt, "The last API call should not throw an exception.");
//
// Final validation
//
Console.WriteLine("Final validation...");
allEntities = this.rtableWrapper.GetAllRows(partitionKey);
m = 0;
int opTypesCounter = 0;
foreach (SampleRTableEntity entity in allEntities)
{
Console.WriteLine("{0}", entity.ToString());
Console.WriteLine("---------------------------------------");
// If the operation is Delete, then skip it. No need to validate.
while (opTypesCounter < count && opTypes[m] == TableOperationType.Delete)
{
m++;
}
Assert.IsTrue(m < count, "m={0} count={1}: m shoud be < count, but it is not.", m, count);
if (opTypes[m] == TableOperationType.Replace)
{
Assert.AreEqual(string.Format(jobType, m), entity.JobType, "JobType does not match");
Assert.AreEqual(string.Format(jobIdTemplate, m), entity.JobId, "JobId does not match");
Assert.AreEqual(string.Format(replaceMessageTemplate, m), entity.Message, "Message does not match");
m++;
}
else
{
throw new ArgumentException(
string.Format("opType={0} is NOT supported", opTypes[opTypesCounter]),
"opType");
}
}
for (int i = 0; i < count; i++)
{
this.ReadFromIndividualAccountsDirectly(
jobType,
string.Format(jobIdTemplate, i),
true);
}
Console.WriteLine("Passed final validation.");
}
public void TableBatchOperationsWithEmptyKeys()
{
CloudTableClient tableClient = GenerateCloudTableClient();
// Insert Entity
DynamicTableEntity ent = new DynamicTableEntity() { PartitionKey = "", RowKey = "" };
ent.Properties.Add("foo2", new EntityProperty("bar2"));
ent.Properties.Add("foo", new EntityProperty("bar"));
TableBatchOperation batch = new TableBatchOperation();
batch.Insert(ent);
currentTable.ExecuteBatch(batch);
// Retrieve Entity
TableBatchOperation retrieveBatch = new TableBatchOperation();
retrieveBatch.Retrieve(ent.PartitionKey, ent.RowKey);
TableResult result = currentTable.ExecuteBatch(retrieveBatch).First();
DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey);
Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey);
Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(ent.Properties["foo"].StringValue, retrievedEntity.Properties["foo"].StringValue);
Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]);
Assert.AreEqual(ent.Properties["foo2"].StringValue, retrievedEntity.Properties["foo2"].StringValue);
Assert.AreEqual(ent.Properties["foo2"], retrievedEntity.Properties["foo2"]);
// InsertOrMerge
DynamicTableEntity insertOrMergeEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey);
insertOrMergeEntity.Properties.Add("foo3", new EntityProperty("value"));
batch = new TableBatchOperation();
batch.InsertOrMerge(insertOrMergeEntity);
currentTable.ExecuteBatch(batch);
result = currentTable.ExecuteBatch(retrieveBatch).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(insertOrMergeEntity.Properties["foo3"], retrievedEntity.Properties["foo3"]);
// InsertOrReplace
DynamicTableEntity insertOrReplaceEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey);
insertOrReplaceEntity.Properties.Add("prop2", new EntityProperty("otherValue"));
batch = new TableBatchOperation();
batch.InsertOrReplace(insertOrReplaceEntity);
currentTable.ExecuteBatch(batch);
result = currentTable.ExecuteBatch(retrieveBatch).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(1, retrievedEntity.Properties.Count);
Assert.AreEqual(insertOrReplaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]);
// Merge
DynamicTableEntity mergeEntity = new DynamicTableEntity(retrievedEntity.PartitionKey, retrievedEntity.RowKey) { ETag = retrievedEntity.ETag };
mergeEntity.Properties.Add("mergeProp", new EntityProperty("merged"));
batch = new TableBatchOperation();
batch.Merge(mergeEntity);
currentTable.ExecuteBatch(batch);
// Retrieve Entity & Verify Contents
result = currentTable.ExecuteBatch(retrieveBatch).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(mergeEntity.Properties["mergeProp"], retrievedEntity.Properties["mergeProp"]);
// Replace
DynamicTableEntity replaceEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey) { ETag = retrievedEntity.ETag };
replaceEntity.Properties.Add("replaceProp", new EntityProperty("replace"));
batch = new TableBatchOperation();
batch.Replace(replaceEntity);
currentTable.ExecuteBatch(batch);
// Retrieve Entity & Verify Contents
result = currentTable.ExecuteBatch(retrieveBatch).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["replaceProp"], retrievedEntity.Properties["replaceProp"]);
// Delete Entity
batch = new TableBatchOperation();
batch.Delete(retrievedEntity);
currentTable.ExecuteBatch(batch);
// Retrieve Entity
result = currentTable.ExecuteBatch(retrieveBatch).First();
Assert.IsNull(result.Result);
}
public void BatchOperationExceptionWhenUsingSmallerViewId()
{
long currentViewId = 100;
long badViewId = currentViewId - 1;
this.UpdateConfiguration(replicas, 0, false, currentViewId);
string jobType = "jobType-BatchOperationExceptionWhenUsingSmallerViewId";
string jobId = "jobId-BatchOperationExceptionWhenUsingSmallerViewId";
int count = 3; // number of operations in the batch _rtable_Operation
List<TableOperationType> opTypes = new List<TableOperationType>()
{
TableOperationType.Replace,
TableOperationType.InsertOrReplace,
TableOperationType.Delete,
};
//
// Insert
//
string jobIdTemplate = jobId + "-{0}";
string messageTemplate = "message-{0}";
string updatedMessageTemplate = "updated-" + messageTemplate;
string partitionKey = string.Empty;
//
// Insert entities
//
for (int i = 0; i < count; i++)
{
SampleRTableEntity originalEntity = new SampleRTableEntity(
jobType,
string.Format(jobIdTemplate, i),
string.Format(messageTemplate, i));
this.repTable.Execute(TableOperation.Insert(originalEntity));
partitionKey = originalEntity.PartitionKey;
}
//
// Retrieve entities and use them to create batchOperation to Replace or Delete
//
IEnumerable<SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey);
TableBatchOperation batchOperation = new TableBatchOperation();
int m = 0;
foreach (SampleRTableEntity entity in allEntities)
{
Console.WriteLine("{0}", entity.ToString());
Console.WriteLine("---------------------------------------");
if (opTypes[m] == TableOperationType.Replace)
{
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(updatedMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
batchOperation.Replace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.InsertOrReplace)
{
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(updatedMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
batchOperation.InsertOrReplace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.Delete)
{
entity.ETag = entity._rtable_Version.ToString();
batchOperation.Delete(entity);
}
else
{
throw new ArgumentException(
string.Format("opType={0} is NOT supported", opTypes[m]),
"opType");
}
m++;
}
//
// Call ModifyConfigurationBlob to change the viewId of the wrapper to an older value
//
Console.WriteLine("Changing the viewId to badViewId {0}", badViewId);
this.UpdateConfiguration(replicas, 0, false, badViewId);
//
// Execute Batch _rtable_Operation with bad viewId
//
Console.WriteLine("\nCalling BatchOperation with badViewId...");
try
{
this.repTable.ExecuteBatch(batchOperation);
}
catch (ReplicatedTableStaleViewException ex)
{
Console.WriteLine("Get this RTableStaleViewException: {0}", ex.Message);
Assert.IsTrue(ex.Message.Contains(string.Format("current _rtable_ViewId {0} is smaller than", badViewId)), "Got unexpected exception message");
}
}
public async Task TableBatchOnSecondaryAsync()
{
AssertSecondaryEndpoint();
CloudTable table = GenerateCloudTableClient().GetTableReference(GenerateRandomTableName());
TableRequestOptions options = new TableRequestOptions()
{
LocationMode = LocationMode.SecondaryOnly,
RetryPolicy = new NoRetry(),
};
TableBatchOperation batch = new TableBatchOperation();
batch.Retrieve("PartitionKey", "RowKey");
OperationContext context = new OperationContext();
await table.ExecuteBatchAsync(batch, options, context);
Assert.AreEqual(StorageLocation.Secondary, context.LastResult.TargetLocation);
batch = new TableBatchOperation();
batch.Insert(new DynamicTableEntity("PartitionKey", "RowKey"));
StorageException e = await TestHelper.ExpectedExceptionAsync<StorageException>(
async () => await table.ExecuteBatchAsync(batch, options, null),
"Batch operations other than retrieve should not be sent to secondary");
Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message);
batch = new TableBatchOperation();
batch.InsertOrMerge(new DynamicTableEntity("PartitionKey", "RowKey"));
e = await TestHelper.ExpectedExceptionAsync<StorageException>(
async () => await table.ExecuteBatchAsync(batch, options, null),
"Batch operations other than retrieve should not be sent to secondary");
Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message);
batch = new TableBatchOperation();
batch.InsertOrReplace(new DynamicTableEntity("PartitionKey", "RowKey"));
e = await TestHelper.ExpectedExceptionAsync<StorageException>(
async () => await table.ExecuteBatchAsync(batch, options, null),
"Batch operations other than retrieve should not be sent to secondary");
Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message);
batch = new TableBatchOperation();
batch.Merge(new DynamicTableEntity("PartitionKey", "RowKey") { ETag = "*" });
e = await TestHelper.ExpectedExceptionAsync<StorageException>(
async () => await table.ExecuteBatchAsync(batch, options, null),
"Batch operations other than retrieve should not be sent to secondary");
Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message);
batch = new TableBatchOperation();
batch.Replace(new DynamicTableEntity("PartitionKey", "RowKey") { ETag = "*" });
e = await TestHelper.ExpectedExceptionAsync<StorageException>(
async () => await table.ExecuteBatchAsync(batch, options, null),
"Batch operations other than retrieve should not be sent to secondary");
Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message);
batch = new TableBatchOperation();
batch.Delete(new DynamicTableEntity("PartitionKey", "RowKey") { ETag = "*" });
e = await TestHelper.ExpectedExceptionAsync<StorageException>(
async () => await table.ExecuteBatchAsync(batch, options, null),
"Batch operations other than retrieve should not be sent to secondary");
Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message);
}
protected override async Task DeleteMessagesToAsync(string persistenceId, long toSequenceNr, bool isPermanent)
{
try
{
CloudTable table = _extension.TableJournalSettings
.GetClient(persistenceId)
.GetTableReference(_extension.TableJournalSettings.TableName);
IEnumerable<Event> results =
table.ExecuteQuery(BuildDeleteTableQuery(persistenceId, toSequenceNr))
.OrderByDescending(t => t.SequenceNr);
if (results.Any())
{
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (Event s in results)
{
s.IsDeleted = true;
if(isPermanent)
{
batchOperation.Delete(s);
}
else
{
batchOperation.Replace(s);
}
}
await table.ExecuteBatchAsync(batchOperation);
}
}
catch (Exception ex)
{
_log.Error(ex, ex.Message);
throw;
}
}
private async Task DoTableBatchReplaceFailAsync(TablePayloadFormat format)
{
tableClient.DefaultRequestOptions.PayloadFormat = format;
// Insert Entity
DynamicTableEntity baseEntity = new DynamicTableEntity("merge test", "foo" + format.ToString());
baseEntity.Properties.Add("prop1", new EntityProperty("value1"));
await currentTable.ExecuteAsync(TableOperation.Insert(baseEntity));
string staleEtag = baseEntity.ETag;
// update entity to rev etag
baseEntity.Properties["prop1"].StringValue = "updated value";
await currentTable.ExecuteAsync(TableOperation.Replace(baseEntity));
OperationContext opContext = new OperationContext();
try
{
// Attempt a merge with stale etag
DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = staleEtag };
replaceEntity.Properties.Add("prop2", new EntityProperty("value2"));
TableBatchOperation batch = new TableBatchOperation();
batch.Replace(replaceEntity);
await currentTable.ExecuteBatchAsync(batch, null, opContext);
Assert.Fail();
}
catch (Exception)
{
TestHelper.ValidateResponse(opContext,
1,
(int)HttpStatusCode.PreconditionFailed,
new string[] { "UpdateConditionNotSatisfied", "ConditionNotMet" },
new string[] { "The update condition specified in the request was not satisfied.", "The condition specified using HTTP conditional header(s) is not met." });
}
// Delete Entity
await currentTable.ExecuteAsync(TableOperation.Delete(baseEntity));
opContext = new OperationContext();
// try replacing with deleted entity
try
{
DynamicTableEntity replaceEntity = new DynamicTableEntity(baseEntity.PartitionKey, baseEntity.RowKey) { ETag = baseEntity.ETag };
replaceEntity.Properties.Add("prop2", new EntityProperty("value2"));
TableBatchOperation batch = new TableBatchOperation();
batch.Replace(replaceEntity);
await currentTable.ExecuteBatchAsync(batch, null, opContext);
Assert.Fail();
}
catch (Exception)
{
TestHelper.ValidateResponse(opContext, 1, (int)HttpStatusCode.NotFound, new string[] { "ResourceNotFound" }, "The specified resource does not exist.");
}
}
private async Task DoEscapeTestAsync(string data, bool useBatch, bool includeKey)
{
DynamicTableEntity ent = new DynamicTableEntity(includeKey ? "temp" + data : "temp", Guid.NewGuid().ToString());
ent.Properties.Add("foo", new EntityProperty(data));
// Insert
if (useBatch)
{
TableBatchOperation batch = new TableBatchOperation();
batch.Insert(ent);
await currentTable.ExecuteBatchAsync(batch);
}
else
{
await currentTable.ExecuteAsync(TableOperation.Insert(ent));
}
// Retrieve
TableResult res = null;
if (useBatch)
{
TableBatchOperation batch = new TableBatchOperation();
batch.Retrieve(ent.PartitionKey, ent.RowKey);
res = (await currentTable.ExecuteBatchAsync(batch))[0];
}
else
{
res = await currentTable.ExecuteAsync(TableOperation.Retrieve(ent.PartitionKey, ent.RowKey));
}
// Check equality
DynamicTableEntity retrievedEntity = res.Result as DynamicTableEntity;
Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey);
Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey);
Assert.AreEqual(ent.ETag, retrievedEntity.ETag);
Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]);
// Merge
ent.Properties.Add("foo2", new EntityProperty("bar2"));
if (useBatch)
{
TableBatchOperation batch = new TableBatchOperation();
batch.Merge(ent);
await currentTable.ExecuteBatchAsync(batch);
}
else
{
await currentTable.ExecuteAsync(TableOperation.Merge(ent));
}
// Retrieve
if (useBatch)
{
TableBatchOperation batch = new TableBatchOperation();
batch.Retrieve(ent.PartitionKey, ent.RowKey);
res = (await currentTable.ExecuteBatchAsync(batch))[0];
}
else
{
res = await currentTable.ExecuteAsync(TableOperation.Retrieve(ent.PartitionKey, ent.RowKey));
}
retrievedEntity = res.Result as DynamicTableEntity;
Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey);
Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey);
Assert.AreEqual(ent.ETag, retrievedEntity.ETag);
Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]);
// Replace
ent.Properties.Remove("foo2");
ent.Properties.Add("foo3", new EntityProperty("bar3"));
if (useBatch)
{
TableBatchOperation batch = new TableBatchOperation();
batch.Replace(ent);
await currentTable.ExecuteBatchAsync(batch);
}
else
{
await currentTable.ExecuteAsync(TableOperation.Replace(ent));
}
// Retrieve
if (useBatch)
{
TableBatchOperation batch = new TableBatchOperation();
batch.Retrieve(ent.PartitionKey, ent.RowKey);
res = (await currentTable.ExecuteBatchAsync(batch))[0];
}
else
{
res = await currentTable.ExecuteAsync(TableOperation.Retrieve(ent.PartitionKey, ent.RowKey));
}
retrievedEntity = res.Result as DynamicTableEntity;
Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey);
Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey);
Assert.AreEqual(ent.ETag, retrievedEntity.ETag);
Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]);
}
private async Task DoTableBatchOperationsWithEmptyKeysAsync(TablePayloadFormat format)
{
tableClient.DefaultRequestOptions.PayloadFormat = format;
// Insert Entity
DynamicTableEntity ent = new DynamicTableEntity() { PartitionKey = "", RowKey = "" };
ent.Properties.Add("foo2", new EntityProperty("bar2"));
ent.Properties.Add("foo", new EntityProperty("bar"));
TableBatchOperation batch = new TableBatchOperation();
batch.Insert(ent);
await currentTable.ExecuteBatchAsync(batch);
// Retrieve Entity
TableBatchOperation retrieveBatch = new TableBatchOperation();
retrieveBatch.Retrieve(ent.PartitionKey, ent.RowKey);
TableResult result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First();
DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey);
Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey);
Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(ent.Properties["foo"].StringValue, retrievedEntity.Properties["foo"].StringValue);
Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]);
Assert.AreEqual(ent.Properties["foo2"].StringValue, retrievedEntity.Properties["foo2"].StringValue);
Assert.AreEqual(ent.Properties["foo2"], retrievedEntity.Properties["foo2"]);
// InsertOrMerge
DynamicTableEntity insertOrMergeEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey);
insertOrMergeEntity.Properties.Add("foo3", new EntityProperty("value"));
batch = new TableBatchOperation();
batch.InsertOrMerge(insertOrMergeEntity);
await currentTable.ExecuteBatchAsync(batch);
result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(insertOrMergeEntity.Properties["foo3"], retrievedEntity.Properties["foo3"]);
// InsertOrReplace
DynamicTableEntity insertOrReplaceEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey);
insertOrReplaceEntity.Properties.Add("prop2", new EntityProperty("otherValue"));
batch = new TableBatchOperation();
batch.InsertOrReplace(insertOrReplaceEntity);
await currentTable.ExecuteBatchAsync(batch);
result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(1, retrievedEntity.Properties.Count);
Assert.AreEqual(insertOrReplaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]);
// Merge
DynamicTableEntity mergeEntity = new DynamicTableEntity(retrievedEntity.PartitionKey, retrievedEntity.RowKey) { ETag = retrievedEntity.ETag };
mergeEntity.Properties.Add("mergeProp", new EntityProperty("merged"));
batch = new TableBatchOperation();
batch.Merge(mergeEntity);
await currentTable.ExecuteBatchAsync(batch);
// Retrieve Entity & Verify Contents
result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(mergeEntity.Properties["mergeProp"], retrievedEntity.Properties["mergeProp"]);
// Replace
DynamicTableEntity replaceEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey) { ETag = retrievedEntity.ETag };
replaceEntity.Properties.Add("replaceProp", new EntityProperty("replace"));
batch = new TableBatchOperation();
batch.Replace(replaceEntity);
await currentTable.ExecuteBatchAsync(batch);
// Retrieve Entity & Verify Contents
result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["replaceProp"], retrievedEntity.Properties["replaceProp"]);
// Delete Entity
batch = new TableBatchOperation();
batch.Delete(retrievedEntity);
await currentTable.ExecuteBatchAsync(batch);
// Retrieve Entity
result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First();
Assert.IsNull(result.Result);
}
/// <summary>
/// Helper function to create some initial entities and then call the specified batchOperation with HttpMangler enabled.
/// </summary>
/// <param name="count"></param>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="targetStorageAccount"></param>
/// <param name="opTypes"></param>
/// <param name="targetApiExpectedToFail"></param>
/// <param name="checkOriginalEntityUnchanged"></param>
/// <param name="checkStorageAccountsConsistent"></param>
/// <param name="httpManglerStartTime"></param>
/// <param name="skipInitialSessions"></param>
/// <returns>ParitionKey of the initial entities created</returns>
protected string SetupAndRunTemperBatchOperation(
int count,
string jobType,
string jobId,
int targetStorageAccount,
List<TableOperationType> opTypes,
bool targetApiExpectedToFail,
bool checkOriginalEntityUnchanged,
bool checkStorageAccountsConsistent,
out DateTime httpManglerStartTime,
int skipInitialSessions = 0)
{
Assert.IsTrue(0 <= targetStorageAccount && targetStorageAccount < this.actualStorageAccountsUsed.Count,
"SetupAndRunTemperBatchOperation() is called with out-of-range targetStorageAccount={0}", targetStorageAccount);
int index = this.actualStorageAccountsUsed[targetStorageAccount];
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[index];
Console.WriteLine("SetupAndRunTemperBatchOperation(): accountNameToTamper={0} skipInitialSessions={1}",
accountNameToTamper, skipInitialSessions);
Assert.AreEqual(count, opTypes.Count, "count and opTypes.Count should be the same");
//
// Tamper behavior
//
ProxyBehavior[] behaviors = new[]
{
TamperBehaviors.TamperAllRequestsIf(
(session => { session.Abort();}),
skipInitialSessions,
AzureStorageSelectors.TableTraffic().IfHostNameContains(accountNameToTamper + "."))
};
string jobIdTemplate = jobId + "-{0}";
string messageTemplate = "message-{0}";
string updatedMessageTemplate = "updated-" + messageTemplate;
string partitionKey = string.Empty;
//
// Insert entities
//
for (int i = 0; i < count; i++)
{
SampleRTableEntity originalEntity = new SampleRTableEntity(
jobType,
string.Format(jobIdTemplate, i),
string.Format(messageTemplate, i));
this.repTable.Execute(TableOperation.Insert(originalEntity));
partitionKey = originalEntity.PartitionKey;
}
//
// Retrieve entities and use them to create batchOperation to Replace or Delete
//
IEnumerable<SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey);
TableBatchOperation batchOperation = new TableBatchOperation();
int m = 0;
foreach (SampleRTableEntity entity in allEntities)
{
Console.WriteLine("{0}", entity.ToString());
Console.WriteLine("---------------------------------------");
if (opTypes[m] == TableOperationType.Replace)
{
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(updatedMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
batchOperation.Replace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.Delete)
{
entity.ETag = entity._rtable_Version.ToString();
batchOperation.Delete(entity);
}
else
{
throw new ArgumentException(
string.Format("opType={0} is NOT supported", opTypes[m]),
"opType");
}
m++;
}
//
// Enable HttpMangler
// Call this.repTable.ExecuteBatch(batchOperation)
//
this.RunHttpManglerBehaviorHelper(
batchOperation,
behaviors,
targetApiExpectedToFail,
out httpManglerStartTime);
if (checkOriginalEntityUnchanged)
{
Console.WriteLine("Validate originalEntity remain unchanged.");
allEntities = this.rtableWrapper.GetAllRows(partitionKey);
batchOperation = new TableBatchOperation();
m = 0;
foreach (SampleRTableEntity entity in allEntities)
{
Console.WriteLine("{0}", entity.ToString());
Console.WriteLine("---------------------------------------");
Assert.AreEqual(string.Format(jobType, m), entity.JobType, "JobType does not match");
Assert.AreEqual(string.Format(jobIdTemplate, m), entity.JobId, "JobId does not match");
Assert.AreEqual(string.Format(messageTemplate, m), entity.Message, "Message does not match");
m++;
}
Console.WriteLine("Passed validation");
}
//
// After httpMangler is turned off, read from individual accounts...
//
Console.WriteLine("\nAfter httpMangler is turned off, read from individual accounts...");
for (int i = 0; i < count; i++)
{
this.ReadFromIndividualAccountsDirectly(
jobType,
string.Format(jobIdTemplate, i),
checkStorageAccountsConsistent);
}
return partitionKey;
}
async Task <string> UploadHistoryBatch(
string instanceId,
string executionId,
TableBatchOperation historyEventBatch,
StringBuilder historyEventNamesBuffer,
int numberOfTotalEvents,
string eTagValue)
{
// Adding / updating sentinel entity
DynamicTableEntity sentinelEntity = new DynamicTableEntity(instanceId, SentinelRowKey)
{
Properties =
{
["ExecutionId"] = new EntityProperty(executionId),
}
};
if (!string.IsNullOrEmpty(eTagValue))
{
sentinelEntity.ETag = eTagValue;
historyEventBatch.Replace(sentinelEntity);
}
else
{
historyEventBatch.InsertOrReplace(sentinelEntity);
}
Stopwatch stopwatch = Stopwatch.StartNew();
IList <TableResult> tableResultList;
try
{
tableResultList = await this.historyTable.ExecuteBatchAsync(
historyEventBatch,
this.StorageTableRequestOptions,
null);
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed)
{
AnalyticsEventSource.Log.SplitBrainDetected(
this.storageAccountName,
this.taskHubName,
instanceId,
executionId,
historyEventBatch.Count,
numberOfTotalEvents,
historyEventNamesBuffer.ToString(0, historyEventNamesBuffer.Length - 1), // remove trailing comma
stopwatch.ElapsedMilliseconds,
eTagValue);
}
throw;
}
this.stats.StorageRequests.Increment();
this.stats.TableEntitiesWritten.Increment(historyEventBatch.Count);
if (tableResultList != null)
{
for (int i = tableResultList.Count - 1; i >= 0; i--)
{
if (((DynamicTableEntity)tableResultList[i].Result).RowKey == SentinelRowKey)
{
eTagValue = tableResultList[i].Etag;
break;
}
}
}
AnalyticsEventSource.Log.AppendedInstanceHistory(
this.storageAccountName,
this.taskHubName,
instanceId,
executionId,
historyEventBatch.Count,
numberOfTotalEvents,
historyEventNamesBuffer.ToString(0, historyEventNamesBuffer.Length - 1), // remove trailing comma
stopwatch.ElapsedMilliseconds,
this.GetETagValue(instanceId));
return(eTagValue);
}
public static void Run([TimerTrigger("0 0 1 * * *")] TimerInfo myTimer, TraceWriter log)
{
log.Info($"Timer trigger function for CopyBlobReadStatsToTables executed at: {DateTime.Now}");
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["AnalyticsStorage"]);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference("$logs");
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("BlobAnalytics");
if (container.Exists())
{
foreach (CloudBlob analyticsBlob in container.ListBlobs(useFlatBlobListing:true))
{
if (!analyticsBlob.Uri.ToString().Contains(DateTime.Now.ToString("yyyy/MM/dd")))
{
var blobAnalyticsList = new List <BlobAnalytics>();
System.IO.Stream file = analyticsBlob.OpenRead();
var blobReader = new System.IO.StreamReader(file);
using (var csvReader = new CsvReader(blobReader))
{
csvReader.ReadHeaderRecord();
while (csvReader.HasMoreRecords)
{
csvReader.ValueSeparator = ';';
var record = csvReader.ReadDataRecord();
if (record[2].Trim() == "GetBlob" && record[7].Trim() == "anonymous" && record[20].Trim() != "0" && record[12].Contains(ConfigurationManager.AppSettings["PathToTrack"]))
{
var analytics = new BlobAnalytics()
{
RequestUrl = record[11],
RequesterIpAddress = record[15],
PartitionKey = record[12].Replace(ConfigurationManager.AppSettings["PathToTrack"], ""),
UserAgentHeader = record[27],
Referrer = record[28],
RawData = record.ToString(),
DownloadTime = record[1],
RowKey = record[13]
};
blobAnalyticsList.Add(analytics);
var batchOperation = new TableBatchOperation();
batchOperation.Insert(analytics);
TableOperation retrieveOperation = TableOperation.Retrieve <PartitionRowCount>(analytics.PartitionKey, "Count");
TableResult retrievedResult = table.Execute(retrieveOperation);
var updateEntity = (PartitionRowCount)retrievedResult.Result;
if (updateEntity == null)
{
updateEntity = new PartitionRowCount
{
PartitionKey = analytics.PartitionKey,
RowKey = "Count"
};
batchOperation.Insert(updateEntity);
}
else
{
updateEntity.Count++;
batchOperation.Replace(updateEntity);
}
try
{
table.ExecuteBatch(batchOperation);
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode != 409)
{
log.Info($"Exception for {analyticsBlob.Uri}: {ex}");
CloudBlobContainer failedLogsContainer = blobClient.GetContainerReference("failedlogs");
CloudBlockBlob targetBlob = failedLogsContainer.GetBlockBlobReference(analyticsBlob.Name);
targetBlob.StartCopy((CloudBlockBlob)analyticsBlob);
throw;
}
}
}
}
}
file.Close();
#if (DEBUG == false)
analyticsBlob.Delete();
#endif
}
}
}
}