public async Task PersistAsync(ICollection <string> entities)
{
var batch = new TableBatchOperation();
foreach (var url in entities)
{
var rowKey = GetRowKey(url);
var row = await _table.ExecuteAsync(TableOperation.Retrieve(PartitionKey, rowKey, new List <string>()));
if (row.Result == null)
{
var persistedEntity = new PersistedEntity
{
PartitionKey = PartitionKey,
RowKey = rowKey,
Url = url,
Notified = false
};
batch.InsertOrMerge(persistedEntity);
}
}
if (batch.Any())
{
await _table.ExecuteBatchAsync(batch);
}
}
public async Task <bool> TryDeleteObservationAsync(string address)
{
// delete wallet if exists
var existed = await _walletStorage.DeleteIfExistAsync(DepositWalletEntity.Partition(address), DepositWalletEntity.Row());
// if not deleted earlier then delete balances
if (existed)
{
string continuation = null;
do
{
var query = new TableQuery <DepositWalletBalanceEntity>().Where($"PartitionKey eq '{DepositWalletBalanceEntity.Partition(address)}'");
var chunk = await _walletBalanceStorage.GetDataWithContinuationTokenAsync(query, 100, continuation);
var batch = new TableBatchOperation();
continuation = chunk.ContinuationToken;
foreach (var balance in chunk.Entities)
{
batch.Delete(balance);
}
if (batch.Any())
{
await _walletBalanceStorage.DoBatchAsync(batch);
}
} while (!string.IsNullOrEmpty(continuation));
}
return(existed);
}
public async Task PurgeAfterAsync(TimeSpan timeSpan)
{
var table = await InitTableAsync();
var query = GetQuery().OrderBy(nameof(ExceptionEntity.Timestamp));
var continuation = default(TableContinuationToken);
do
{
var segment = await table.ExecuteQuerySegmentedAsync(query, continuation);
var delete = new TableBatchOperation();
foreach (var entity in segment.Results)
{
var age = DateTime.UtcNow.Subtract(entity.Timestamp.UtcDateTime);
if (age > timeSpan)
{
delete.Add(TableOperation.Delete(entity));
}
}
if (delete.Any())
{
await table.ExecuteBatchAsync(delete);
}
continuation = segment.ContinuationToken;
} while (continuation != null);
}
private async Task ConsolidateOrder(Guild guild)
{
if (guild.RoleAssignations == null || !guild.RoleAssignations.Any())
{
await guild.LoadChildrens(g => g.RoleAssignations);
}
if (!guild.RoleAssignations.Any())
{
this._telemetry.TrackEvent("No roles to reorder, finishing");
return;
}
var lastOrder = 0;
var batch = new TableBatchOperation();
foreach (var roleAssignation in guild.RoleAssignations.OrderBy(ra => ra.Order).ToList())
{
roleAssignation.Order = lastOrder;
lastOrder += 1;
batch.Merge(roleAssignation);
}
if (batch.Any())
{
var bindingsTable = GetTable <RoleAssignation>();
await bindingsTable.ExecuteBatchAsync(batch);
}
}
public static async Task Remove <TEntity>(this CloudTable table, IList <TEntity> entities)
where TEntity : ITableEntity, new()
{
if (entities == null)
{
throw new ArgumentNullException(nameof(entities));
}
if (!entities.Any())
{
return;
}
var batch = new TableBatchOperation();
foreach (var entity in entities)
{
batch.Add(TableOperation.Delete(entity));
// Table batch can have up to 100 operations.
if (batch.Count == 100)
{
await table.ExecuteBatchAsync(batch);
batch.Clear();
}
}
if (batch.Any())
{
await table.ExecuteBatchAsync(batch);
}
}
internal async Task ExecuteBatchSafeAsync(TableBatchOperation batch, DateTime?now = null)
{
var metricsTable = GetMetricsTable(now);
if (metricsTable != null && batch.Any())
{
try
{
// TODO: handle paging and errors
await metricsTable.ExecuteBatchAsync(batch);
}
catch (StorageException e)
{
if (IsNotFoundTableNotFound(e))
{
// create the table and retry
await CreateIfNotExistsAsync(metricsTable);
await metricsTable.ExecuteBatchAsync(batch);
return;
}
throw;
}
}
}
public void ClearTable()
{
var getPersonOperation = TableOperation.Retrieve <PersonEntity>(LastName, FirstName);
var tableResult = cloudTable.Execute(getPersonOperation);
var person = (PersonEntity)tableResult.Result;
if (person != null)
{
Assert.AreEqual(LastName, person.LastName());
cloudTable.Execute(TableOperation.Delete(person));
}
TableQuery <AnimalEntity> animalsQuery = new TableQuery <AnimalEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, AnimalEntity.AnimalPartition));
TableBatchOperation tableBatchOperation = new TableBatchOperation();
foreach (AnimalEntity animalEntity in cloudTable.ExecuteQuery(animalsQuery))
{
var tableOperation = TableOperation.Delete(animalEntity);
tableBatchOperation.Add(tableOperation);
}
if (tableBatchOperation.Any())
{
cloudTable.ExecuteBatch(tableBatchOperation);
}
}
public void Save(string partitionKey, IEnumerable <EventData> events)
{
var tableRef = tableClient.GetTableReference(this.tableName);
var batchOperation = new TableBatchOperation();
foreach (var eventData in events)
{
string creationDate = DateTime.UtcNow.ToString("o");
var formattedVersion = eventData.Version.ToString("D10");
var dbEntity =
Mapper.Map(eventData, new EventTableServiceEntity
{
PartitionKey = partitionKey,
RowKey = formattedVersion,
CreationDate = creationDate,
});
batchOperation.Insert(dbEntity);
// Add a duplicate of this event to the Unpublished "queue"
var dbDuplicateEntity =
Mapper.Map(eventData, new EventTableServiceEntity
{
PartitionKey = partitionKey,
RowKey = UnpublishedRowKeyPrefix + formattedVersion,
CreationDate = creationDate,
});
batchOperation.Insert(dbDuplicateEntity);
}
if (!batchOperation.Any())
{
logger.LogWarning($"DB Storage batch is empty for partitionKey {partitionKey}, nothing to save!");
return;
}
try
{
eventStoreRetryPolicy.Execute(() => tableRef.ExecuteBatchAsync(batchOperation).Result);
}
catch (Exception ex)
{
var inner = ex.InnerException as StorageException;
if (inner == null)
{
throw;
}
var hResult = inner.RequestInformation.HttpStatusCode;
if (hResult == (int)HttpStatusCode.Conflict)
{
//throw new ConcurrencyException();
return;
}
throw new ConcurrencyException();
}
}
/// <summary>
/// Removes all entries from a given Azure Storage Table.
/// </summary>
/// <param name="table">The table to clear.</param>
/// <param name="parallelity">The amount of threads to use in parallel</param>
/// <param name="raiseEvents">If <c>true</c> the <see cref="TableItemsRemoved"/> event will be raised regulary to indicate progress.</param>
public static async Task ClearAsync(this CloudTable table, int parallelity = 0, bool raiseEvents = true)
{
var query = new TableQuery <WadLogEntity>();
TableContinuationToken continuationToken = null;
if (parallelity <= 0)
{
parallelity = Environment.ProcessorCount * 4;
}
parallelity = Math.Min(parallelity, 48);
var tasks = new List <Task>();
do
{
var tableQueryResult = await table.ExecuteQuerySegmentedAsync(query, continuationToken);
continuationToken = tableQueryResult.ContinuationToken;
var elements = tableQueryResult.OrderBy(ele => ele.PartitionKey).ToList();
while (tasks.Count == parallelity)
{
Task.WaitAny(tasks.ToArray());
tasks.RemoveAll(t => t.IsCompleted || t.IsCanceled);
}
tasks.Add(
Task.Run(
async() =>
{
var lastPartKey = string.Empty;
var operations = new TableBatchOperation();
foreach (var item in elements)
{
if (!lastPartKey.IsNullOrEmpty() && !item.PartitionKey.Equals(lastPartKey))
{
// we have to start the batch now because the current item will have a new
// partition key and batches are only allowed within the same partition key
await table.ExecuteBatchAsync(operations, raiseEvents);
}
operations.Add(TableOperation.Delete(item));
lastPartKey = item.PartitionKey;
if (operations.Count == 100)
{
// take care that the maximum amount of items for a batch is used
await table.ExecuteBatchAsync(operations, raiseEvents);
}
}
if (operations.Any())
{
// take care of the rest of the operations
await table.ExecuteBatchAsync(operations, raiseEvents);
}
}));
}while (continuationToken != null);
Task.WaitAll(tasks.ToArray());
}
public void Add(IEnumerable <T> obj)
{
var batchOperation = new TableBatchOperation();
foreach (var item in obj)
{
batchOperation.Insert(item);
}
if (batchOperation.Any())
{
_tableClient.ExecuteBatch(batchOperation);
}
}
public static async Task Run(
[QueueTrigger(QueueNames.SyncDay)] SyncDayRequest request,
[Table(TableNames.AthletesTable, "{AthleteId}", "")] AthleteTableEntity athlete,
[Table(TableNames.AthletesTable)] CloudTable athletesTable,
[Table(TableNames.SyncedActivitiesTable)] CloudTable syncedActivitiesTable,
ILogger log)
{
var client = new LittleStravaClient();
var date = request.DateTime.Date;
var before = new DateTimeOffset(date.AddDays(1)).ToUnixTimeSeconds();
var after = new DateTimeOffset(date).ToUnixTimeSeconds();
var activities = await client.GetLoggedInAthleteActivities(athlete, before, after);
var types = athlete.TypesToSync.Split(",").Select(Enum.Parse <LittleStravaClient.ActivityType>).ToArray();
var activitiesToSync = activities.Where(x => types.Contains(x.Type))
.Select(x => new { x.Id, x.Type, Meters = x.Distance, Miles = ConvertToMiles(x.Distance) })
.ToArray();
var totalDaysMiles = activitiesToSync.Sum(x => x.Miles);
if (totalDaysMiles > 0)
{
var landsEnd3FireBaseAppClient = new LandsEnd3FireBaseAppClient();
await landsEnd3FireBaseAppClient.SubmitData(athlete, date, totalDaysMiles);
var tableBatchOperation = new TableBatchOperation();
foreach (var sycnedActivity in activitiesToSync)
{
tableBatchOperation.InsertOrReplace(new SyncedActivitiesTableEntity
{
PartitionKey = request.AthleteId.ToString(),
RowKey = sycnedActivity.Id.ToString(),
ActivityType = sycnedActivity.Type.ToString(),
Meters = sycnedActivity.Meters,
Miles = sycnedActivity.Miles,
});
}
if (tableBatchOperation.Any())
{
syncedActivitiesTable.ExecuteBatch(tableBatchOperation);
}
}
await athletesTable.ExecuteAsync(TableOperation.InsertOrReplace(athlete));
}
private void DeleteIt(IEnumerable <TEntityType> items)
{
var tableBatchOperation = new TableBatchOperation();
foreach (var item in items.Where(t => t != null))
{
item.ETag = "*";
tableBatchOperation.Add(TableOperation.Delete(item));
}
if (!tableBatchOperation.Any())
{
return;
}
// ReSharper disable once UnusedVariable
var result = Execute(tableBatchOperation).Result;
}
/// <summary>
/// Method for Clearing Role Status table
/// </summary>
/// <param name="partitionKey">Namespace of the WorkerRole (eg. Crawler)</param>
private void ClearRoleStatusTableContent(string partitionKey)
{
TableQuery <RoleStatus> query = new TableQuery <RoleStatus>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey))
.Select(new List <string> {
"PartitionKey", "RowKey"
});
var entities = roleStatusTable.ExecuteQuery(query);
TableBatchOperation deleteOldeRecords = new TableBatchOperation();
foreach (var entity in entities)
{
deleteOldeRecords.Delete(entity);
}
if (deleteOldeRecords.Any())
{
roleStatusTable.ExecuteBatchAsync(deleteOldeRecords);
}
}
public async Task DeletePartitionAsync <T>(string tableName, string partitionKey) where T : class, ITableEntity, new()
{
CloudTable table = tableClient.GetTableReference(tableName);
var partitionObjects = await RetrievePartitionFromTableAsync <T>(tableName, partitionKey);
if (table != null)
{
var batch = new TableBatchOperation();
foreach (var obj in partitionObjects)
{
batch.Delete(obj);
}
if (batch.Any())
{
await table.ExecuteBatchAsync(batch);
}
}
}
private void ExecuteBatchSafely(CloudTable table, TableBatchOperation batch)
{
if (batch == null || !batch.Any())
{
return;
}
try
{
table.ExecuteBatch(batch);
}
catch (StorageException ex)
{
if (ex.RequestInformation.ExtendedErrorInformation.ErrorCode == "ResourceNotFound")
{
if (batch.Count == 1)
{
return;
}
TableBatchOperation firstBatch = new TableBatchOperation();
TableBatchOperation secondBatch = new TableBatchOperation();
for (int i = 0; i < batch.Count; i++)
{
if (i % 2 == 0)
{
firstBatch.Add(batch[i]);
}
else
{
secondBatch.Add(batch[i]);
}
}
ExecuteBatchSafely(table, firstBatch);
ExecuteBatchSafely(table, secondBatch);
}
}
catch (Exception ex)
{
_logger.Write(ex);
}
}
/// <summary>
/// Executes a batch operation on a Azure Storage table securely and raises the <see cref="TableItemsRemoved"/> event.
/// </summary>
/// <remarks>
/// Be ware to supply only operations that are working on the same partition key. Otherwise an exception will occur in
/// Azure Storage.
/// </remarks>
/// <param name="table">The table on which to execute the batch.</param>
/// <param name="operations">The batch containing the operations.</param>
/// <param name="raiseEvents">If <c>true</c> the <see cref="TableItemsRemoved"/> event will be raised if the btach completed.</param>
public static async Task ExecuteBatchAsync(this CloudTable table, TableBatchOperation operations, bool raiseEvents)
{
if (!operations.Any())
{
return;
}
try
{
await table.ExecuteBatchAsync(operations);
if (raiseEvents)
{
TableItemsRemoved?.Invoke(null, new AmountBasedEventArgs(operations.Count));
}
operations.Clear();
}
catch (Exception ex)
{
Trace.TraceError(ex.Message);
}
}
public async Task MarkNotifiedAsync(ICollection <string> entities)
{
var batch = new TableBatchOperation();
foreach (var url in entities)
{
var persistedEntity = new PersistedEntity
{
PartitionKey = PartitionKey,
RowKey = GetRowKey(url),
Url = url,
Notified = true
};
batch.InsertOrMerge(persistedEntity);
}
if (batch.Any())
{
await _table.ExecuteBatchAsync(batch);
}
}
public async Task <bool> RemoveSubscriptions(IEnumerable <UserSubscription> subscriptionsToRemove)
{
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable userSubscriptionsTable = tableClient.GetTableReference("userSubscriptions");
var tableExists = await userSubscriptionsTable.ExistsAsync();
if (!tableExists)
{
return(false);
}
List <UserSubscriptionEntity> activeUserSubscriptionEntities = new List <UserSubscriptionEntity>();
Expression <Func <UserSubscriptionEntity, bool> > filter =
(x) => x.PartitionKey == subscriptionsToRemove.First().UserId.ToString();
Action <IEnumerable <UserSubscriptionEntity> > processor = activeUserSubscriptionEntities.AddRange;
await ObtainUserSubscriptionEntities(userSubscriptionsTable, filter, processor);
TableBatchOperation deletionBatchOperation = new TableBatchOperation();
foreach (var userSubscription in subscriptionsToRemove)
{
var entity = activeUserSubscriptionEntities.SingleOrDefault(
x => x.PartitionKey == userSubscription.UserId.ToString() &&
x.RowKey == userSubscription.FriendId.ToString());
if (entity != null)
{
deletionBatchOperation.Add(TableOperation.Delete(entity));
}
}
if (deletionBatchOperation.Any())
{
await userSubscriptionsTable.ExecuteBatchAsync(deletionBatchOperation);
}
return(true);
}
/// <summary>
/// Insert multiple records
/// </summary>
/// <param name="records">The records to insert</param>
public void Insert <T>(IEnumerable <T> records) where T : class, ITableEntity
{
if (records == null)
{
throw new ArgumentNullException(nameof(records));
}
var partitionKeySeparation = records.GroupBy(x => x.PartitionKey)
.OrderBy(g => g.Key)
.Select(g => g.AsEnumerable()).SelectMany(entry => entry.Partition(MaxPartitionSize)).ToList();
foreach (var entry in partitionKeySeparation)
{
var operation = new TableBatchOperation();
entry.ToList().ForEach(operation.Insert);
if (operation.Any())
{
CloudTable.ExecuteBatch(operation);
}
}
}
public async Task InsertOrMerge(IEnumerable <T> records)
{
if (records == null)
{
throw new ArgumentNullException(nameof(records));
}
var partitionSeparation = records.GroupBy(x => x.PartitionKey)
.OrderBy(g => g.Key)
.Select(g => g.ToList());
foreach (var entry in partitionSeparation)
{
var operation = new TableBatchOperation();
entry.ForEach(operation.InsertOrMerge);
if (operation.Any())
{
await _cloudTable.ExecuteBatchAsync(operation);
}
}
}
/// <summary>
/// Insert multiple records
/// </summary>
/// <param name="records">The records to insert</param>
public async Task InsertAsync(IEnumerable <T> records)
{
if (records == null)
{
throw new ArgumentNullException(nameof(records));
}
var partitionKeySeparation = records.GroupBy(x => x.PartitionKey)
.OrderBy(g => g.Key)
.Select(g => g.AsEnumerable()).SelectMany(entry => entry.Partition(MaxPartitionSize)).ToList();
foreach (var entry in partitionKeySeparation)
{
var operation = new TableBatchOperation();
entry.ToList().ForEach(operation.Insert);
if (operation.Any())
{
await _cloudTable.ExecuteBatchAsync(operation).ConfigureAwait(false);
}
}
}
private async Task SaveMovies(IEnumerable <SearchResult> movies, string channelId, CloudTable table)
{
var operation = new TableBatchOperation();
foreach (var movie in movies)
{
_log.Info("Saving new movie: " + movie.Snippet.Title);
operation.Insert(new MovieRow()
{
PartitionKey = GetPartitionKey(channelId),
RowKey = movie.Id.VideoId,
Text = movie.Snippet.Title
});
}
if (operation.Any())
{
await table.ExecuteBatchAsync(operation);
_log.Info("Saving new movies sucessed");
}
}
public async Task SaveAsync()
{
if (!Validate())
{
return;
}
foreach (var item in Customers.Where(c => c.RowKey != Guid.Empty))
{
var entity = TableCustomers.FirstOrDefault(c => c.PartitionKey == item.LastName && c.RowKey == item.RowKey.ToString());
entity.FirstName = item.FirstName;
entity.EmailAddress = item.EmailAddress;
entity.PhoneNumber = item.PhoneNumber;
var update = TableOperation.Replace(entity);
await customersTable.ExecuteAsync(update);
}
var batchOperation = new TableBatchOperation();
foreach (var item in Customers.Where(c => c.RowKey == Guid.Empty))
{
var customer = new CustomerEntity(item.LastName);
customer.FirstName = item.FirstName;
customer.EmailAddress = item.EmailAddress;
customer.PhoneNumber = item.PhoneNumber;
batchOperation.Insert(customer);
item.RowKey = Guid.Parse(customer.RowKey);
TableCustomers.Add(customer);
}
if (batchOperation.Any())
{
await customersTable.ExecuteBatchAsync(batchOperation);
}
Message = "";
}
public async Task<bool> RemoveSubscriptions(IEnumerable<UserSubscription> subscriptionsToRemove)
{
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable userSubscriptionsTable = tableClient.GetTableReference("userSubscriptions");
var tableExists = await userSubscriptionsTable.ExistsAsync();
if (!tableExists)
{
return false;
}
List<UserSubscriptionEntity> activeUserSubscriptionEntities = new List<UserSubscriptionEntity>();
Expression<Func<UserSubscriptionEntity, bool>> filter =
(x) => x.PartitionKey == subscriptionsToRemove.First().UserId.ToString();
Action<IEnumerable<UserSubscriptionEntity>> processor = activeUserSubscriptionEntities.AddRange;
await ObtainUserSubscriptionEntities(userSubscriptionsTable, filter, processor);
TableBatchOperation deletionBatchOperation = new TableBatchOperation();
foreach (var userSubscription in subscriptionsToRemove)
{
var entity = activeUserSubscriptionEntities.SingleOrDefault(
x => x.PartitionKey == userSubscription.UserId.ToString() &&
x.RowKey == userSubscription.FriendId.ToString());
if (entity != null)
{
deletionBatchOperation.Add(TableOperation.Delete(entity));
}
}
if (deletionBatchOperation.Any())
{
await userSubscriptionsTable.ExecuteBatchAsync(deletionBatchOperation);
}
return true;
}
public void ClearTable()
{
var getPersonOperation = TableOperation.Retrieve<PersonEntity>(LastName, FirstName);
var tableResult = cloudTable.Execute(getPersonOperation);
var person = (PersonEntity)tableResult.Result;
if (person != null)
{
Assert.AreEqual(LastName, person.LastName());
cloudTable.Execute(TableOperation.Delete(person));
}
TableQuery<AnimalEntity> animalsQuery = new TableQuery<AnimalEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, AnimalEntity.AnimalPartition));
TableBatchOperation tableBatchOperation = new TableBatchOperation();
foreach (AnimalEntity animalEntity in cloudTable.ExecuteQuery(animalsQuery))
{
var tableOperation = TableOperation.Delete(animalEntity);
tableBatchOperation.Add(tableOperation);
}
if (tableBatchOperation.Any())
{
cloudTable.ExecuteBatch(tableBatchOperation);
}
}
private static int TruncateTable(CloudTable destinationTable)
{
// get all destination rows sorted by keys for faster operations
__("Reading table...");
var query = new TableQuery {
SelectColumns = new List <string>()
};
var destinationEntityList = destinationTable.ExecuteQuery(query).ToList();
var destinationEntityGroups = destinationEntityList
.GroupBy(o => o.PartitionKey)
.ToList();
__("Got {0} entities partitioned into {1} groups",
destinationEntityList.Count, destinationEntityGroups.Count);
// delete by batches of 100 / on PK change
var batchOperation = new TableBatchOperation();
var batchOperationList = new List <TableBatchOperation>();
foreach (var entityGroup in destinationEntityGroups)
{
// enumerate group
var entities = entityGroup.ToList();
// work with 100 operations at a time
foreach (var entity in entities)
{
if (batchOperation.Any() && batchOperation.Count % 100 == 0)
{
// add and reset batch
batchOperationList.Add(batchOperation);
batchOperation = new TableBatchOperation();
}
// add operation to batch
batchOperation.Delete(entity);
}
// add last batch
if (batchOperation.Count > 0)
{
batchOperationList.Add(batchOperation);
batchOperation = new TableBatchOperation();
}
}
// process batches
var batchTaskList = batchOperationList
.Select(destinationTable.ExecuteBatchAsync)
.Cast <Task>()
.ToList();
// wait for all operations to complete
__("Executing {0} delete batches", batchTaskList.Count);
while (batchTaskList.Count(o => !o.IsCompleted) > 0)
{
Thread.Sleep(250);
}
return(destinationEntityList.Count);
}
