public async Task ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> messages)
{
var batch = new TableBatchOperation();
foreach(var msg in messages)
{
var snap = JsonConvert.DeserializeObject<BusSnapshotInfo>(Encoding.UTF8.GetString(msg.GetBytes()));
var entity = new DynamicTableEntity(snap.RouteShortName, snap.VehicleId.ToString());
entity.Properties.Add("RouteShortName", EntityProperty.GeneratePropertyForString(snap.RouteShortName));
entity.Properties.Add("VehicleId", EntityProperty.GeneratePropertyForInt(snap.VehicleId));
entity.Properties.Add("TripId", EntityProperty.GeneratePropertyForInt(snap.TripId));
entity.Properties.Add("Latitude", EntityProperty.GeneratePropertyForDouble(snap.Latitude));
entity.Properties.Add("Longitude", EntityProperty.GeneratePropertyForDouble(snap.Longitude));
entity.Properties.Add("DirectionOfTravel", EntityProperty.GeneratePropertyForString(snap.DirectionOfTravel.ToString()));
entity.Properties.Add("NextStopId", EntityProperty.GeneratePropertyForInt(snap.NextStopId));
entity.Properties.Add("Timeliness", EntityProperty.GeneratePropertyForString(snap.Timeliness.ToString()));
entity.Properties.Add("TimelinessOffset", EntityProperty.GeneratePropertyForInt(snap.TimelinessOffset));
entity.Properties.Add("Timestamp", EntityProperty.GeneratePropertyForDateTimeOffset(snap.Timestamp));
batch.Add(TableOperation.InsertOrReplace(entity));
}
var tableClient = _account.CreateCloudTableClient();
var table = tableClient.GetTableReference("snapshots");
await table.CreateIfNotExistsAsync();
await table.ExecuteBatchAsync(batch);
await context.CheckpointAsync();
}
private static void PopulateTableData(CloudTable cloudTable)
{
// if the table does not exist then create it and populate it wih some data
if (!cloudTable.Exists())
{
cloudTable.CreateIfNotExists();
var tableBatchOperation = new TableBatchOperation();
for (int i = 0; i < 100; i++)
{
tableBatchOperation.Add(
TableOperation.Insert(new Person(i.ToString(), string.Format("Person {0}", i))));
}
cloudTable.ExecuteBatch(tableBatchOperation);
}
}
static void CreateCustomerMetadata(CloudTableClient tableClient)
{
Console.WriteLine("Creating customers metadata...");
CloudTable customersMetadataTable = tableClient.GetTableReference("customersmetadata");
customersMetadataTable.CreateIfNotExists();
var msftAddress1 = new DictionaryTableEntity();
msftAddress1.PartitionKey = "MSFT";
msftAddress1.RowKey = "ADDRESS-" + Guid.NewGuid().ToString("N").ToUpper();
msftAddress1.Add("city", "Seattle");
msftAddress1.Add("street", "111 South Jackson");
var msftWebsite1 = new DictionaryTableEntity();
msftWebsite1.PartitionKey = "MSFT";
msftWebsite1.RowKey = "WEBSITE-" + Guid.NewGuid().ToString("N").ToUpper();
msftWebsite1.Add("url", "http://www.microsoft.com");
var msftWebsite2 = new DictionaryTableEntity();
msftWebsite2.PartitionKey = "MSFT";
msftWebsite2.RowKey = "WEBSITE-" + Guid.NewGuid().ToString("N").ToUpper();
msftWebsite2.Add("url", "http://www.windowsazure.com");
var batch = new TableBatchOperation();
batch.Add(TableOperation.Insert(msftAddress1));
batch.Add(TableOperation.Insert(msftWebsite1));
batch.Add(TableOperation.Insert(msftWebsite2));
customersMetadataTable.ExecuteBatch(batch);
Console.WriteLine("Done. Press ENTER to read the customer metadata.");
Console.ReadLine();
}
public void AddEntryToTable(string tableName)
{
CloudTable table = cloudTableClient.GetTableReference(tableName);
table.CreateIfNotExists();
TableBatchOperation batch = new TableBatchOperation();
batch.Add(TableOperation.Insert(new TableEntity { RowKey = "Abhishek", PartitionKey = "Kolkata" }));
batch.Add(TableOperation.Insert(new TableEntity { RowKey = "Abhijit", PartitionKey = "Kolkata" }));
table.ExecuteBatch(batch);
}
internal async Task <Tuple <string, string> > InsertTwoTableEntriesConditionallyAsync(T data1, T data2, string data2Etag)
{
const string operation = "InsertTableEntryConditionally";
string data2Str = (data2 == null ? "null" : data2.ToString());
var startTime = DateTime.UtcNow;
if (Logger.IsVerbose2)
{
Logger.Verbose2("{0} into table {1} data1 {2} data2 {3}", operation, TableName, data1, data2Str);
}
try
{
try
{
// WAS:
// Only AddObject, do NOT AttachTo. If we did both UpdateObject and AttachTo, it would have been equivalent to InsertOrReplace.
// svc.AddObject(TableName, data);
// ---
// svc.AttachTo(TableName, tableVersion, tableVersionEtag);
// svc.UpdateObject(tableVersion);
// SaveChangesOptions.ReplaceOnUpdate | SaveChangesOptions.Batch,
// EntityDescriptor dataResult = svc.GetEntityDescriptor(data);
// return dataResult.ETag;
var entityBatch = new TableBatchOperation();
entityBatch.Add(TableOperation.Insert(data1));
data2.ETag = data2Etag;
entityBatch.Add(TableOperation.Replace(data2));
var opResults = await Task <IList <TableResult> > .Factory.FromAsync(
tableReference.BeginExecuteBatch,
tableReference.EndExecuteBatch,
entityBatch,
null);
//The batch results are returned in order of execution,
//see reference at https://msdn.microsoft.com/en-us/library/microsoft.windowsazure.storage.table.cloudtable.executebatch.aspx.
//The ETag of data is needed in further operations.
return(new Tuple <string, string>(opResults[0].Etag, opResults[1].Etag));
}
catch (Exception exc)
{
CheckAlertWriteError(operation, data1, data2Str, exc);
throw;
}
}
finally
{
CheckAlertSlowAccess(startTime, operation);
}
}
private static string TableInsertAlunos(CloudStorageAccount conta)
{
CloudTableClient tableClient = conta.CreateCloudTableClient();
CloudTable jogos = tableClient.GetTableReference("jogos");
jogos.CreateIfNotExists();
Jogo santosFluminense = new Jogo("A", "1")
{
TimeA = "Flu",
TimeB = "SAN",
DataJogo = new DateTime(2017, 5, 13)
};
Jogo palvasco = new Jogo("A", "2")
{
TimeA = "PAL",
TimeB = "VASCO",
DataJogo = new DateTime(2017, 5, 14)
};
Jogo corponte = new Jogo("A", "3")
{
TimeA = "COR",
TimeB = "PONTE",
DataJogo = new DateTime(2017, 5, 14)
};
TableBatchOperation lote = new TableBatchOperation();
lote.Add(TableOperation.Insert(santosFluminense));
lote.Add(TableOperation.Insert(palvasco));
// jogos.ExecuteBatch(lote);
jogos.Execute(TableOperation.Insert(corponte));
SharedAccessTablePolicy regra = new SharedAccessTablePolicy();
regra.SharedAccessStartTime = DateTime.UtcNow.AddMinutes(-5);
regra.SharedAccessExpiryTime = DateTime.UtcNow.AddDays(2);
regra.Permissions =
SharedAccessTablePermissions.Delete
| SharedAccessTablePermissions.Query
| SharedAccessTablePermissions.Update;
var sasToken =
jogos.GetSharedAccessSignature(regra, null, null, null, null, null);
return($"{jogos.Uri}{sasToken}");
}
public async Task RunGarbageCollectionInXTableAsync(CancellationToken cancellationToken)
{
var tsTable = _xTableClient.GetTableReference(_config.Value.AzStorageTable);
var notBeforeTime = DateTime.UtcNow.Subtract(TimeSpan.FromDays(_config.Value.RetentionDays));
var contToken = default(TableContinuationToken);
var gcQuery = new TableQuery <ContribSampleEntity>().Where(
TableQuery.GenerateFilterConditionForDate(
"MetricTimeStampUtc", QueryComparisons.LessThan, notBeforeTime
));
_logger.LogInformation("TS Garbage Collection started");
do
{
var result = await tsTable.ExecuteQuerySegmentedAsync(gcQuery, contToken, cancellationToken);
contToken = result.ContinuationToken;
_logger.LogInformation("TS GC retrieved a batch");
if (result.Results != null)
{
var partitionGroup = result.Results.GroupBy(i => i.PartitionKey);
foreach (var p in partitionGroup)
{
try
{
var batch = new TableBatchOperation();
foreach (var i in p)
{
batch.Add(TableOperation.Delete(i));
if (batch.Count % 50 == 0)
{
await tsTable.ExecuteBatchAsync(batch, cancellationToken);
batch.Clear();
}
}
if (batch.Count > 0)
{
await tsTable.ExecuteBatchAsync(batch, cancellationToken);
}
_logger.LogInformation("TS GC deleted an item");
}
catch (Exception exc)
{
_logger.LogInformation(exc, "TS GC failed to delete a batch but ignored");
}
}
_logger.LogInformation($"TS Garbage Collection deleted batch of {result.Results.Count} item(s)");
}
}while (contToken != null && !cancellationToken.IsCancellationRequested);
_logger.LogInformation("TS Garbage Collection completed");
}
void ClearTable(CloudTable table)
{
var deviceIds = _deviceService.GetDeviceIds();
foreach (var partitionKey in deviceIds)
{
TableBatchOperation batchDelete = new TableBatchOperation();
// gets all the entities in the table for this partition key
string partitionCondition = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
List<DynamicTableEntity> entities = table.ExecuteQuery(new TableQuery().Where(partitionCondition)).ToList();
entities.ForEach(e =>
{
batchDelete.Add(TableOperation.Delete(e));
// Azure has a limit on batch operations
if (batchDelete.Count == 100)
{
table.ExecuteBatch(batchDelete);
batchDelete = new TableBatchOperation();
}
});
// flush out whatever is left
if (batchDelete.Count > 0)
{
table.ExecuteBatch(batchDelete);
}
}
}
public List<Reply> GetReplyNotif(string userid)
{
TableQuery<ReplyNotificationEntifity> query = new TableQuery<ReplyNotificationEntifity>().Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, userid));
List<Reply> replies = new List<Reply>();
TableBatchOperation batchOperation = new TableBatchOperation();
int count = 0;
// Print the fields for each customer.
foreach (ReplyNotificationEntifity entity in _replyNotification.ExecuteQuery(query))
{
replies.Add(JsonConvert.DeserializeObject<Reply>(entity.Content));
batchOperation.Add(TableOperation.Delete(entity));
count++;
if((count % 100) == 0){
_replyNotification.ExecuteBatch(batchOperation);
batchOperation = new TableBatchOperation();
count = 0;
}
}
if (count > 0)
{
_replyNotification.ExecuteBatch(batchOperation);
}
return replies;
}
/// <inheritdoc />
public void Add(IStorageTableOperation operation)
{
TableOperation sdkOperation = ((StorageTableOperation)operation).SdkObject;
_sdk.Add(sdkOperation);
_items.Add(operation);
}
private static async Task <List <UploadResultItem> > Insert(string owner, string partition, CloudTable itemTable, List <ItemJson> items)
{
Debug.Assert(items.Count <= 100);
var ouputKey = partition.Remove(0, owner.Length);
var batch = new TableBatchOperation();
foreach (var item in items)
{
batch.Add(TableOperation.Insert(ItemBuilder.CreateV2(partition, item)));
}
var mapping = new List <UploadResultItem>();
var results = await itemTable.ExecuteBatchAsync(batch);
for (int i = 0; i < items.Count; i++)
{
var saved = results[i].Result as ItemV2;
mapping.Add(new UploadResultItem {
LocalId = items[i].LocalId,
Id = saved.RowKey,
Partition = ouputKey // The email+whatever is only used to represent the item internally
});
}
return(mapping);
}
public static async Task AddRedisAssetUserRelation(CloudTableClient tableClient, List <UserAssetRatio> addAssetUserRatios, string writeAssetUserRatioAzureTable, int batchNum = 100)
{
if (addAssetUserRatios.Any())
{
CloudTable assetuserTable = tableClient.GetTableReference(writeAssetUserRatioAzureTable);
var assetIds = addAssetUserRatios.Select(p => p.AssetId).ToList();
foreach (var item in assetIds)
{
var currentUserAsset = addAssetUserRatios.Where(p => p.AssetId == item).ToList();
if (currentUserAsset.Any())
{
int ratioCount = (int)Math.Ceiling((double)currentUserAsset.Count / batchNum);
for (int i = 0; i < ratioCount; i++)
{
TableBatchOperation batch = new TableBatchOperation();
foreach (UserAssetRatio tableEntity in currentUserAsset.Skip(batchNum * i).Take(batchNum))
{
batch.Add(TableOperation.InsertOrReplace(tableEntity));
}
await assetuserTable.ExecuteBatchAsync(batch);
}
}
}
}
}
private static async Task PersistPngImageFilePaths(ILogger log, IEnumerable <PngImage> pngImages)
{
if (!pngImages.Any())
{
log.LogInformation("No PNG images to persist.");
return;
}
var storageAccount = CloudStorageAccount.Parse(Environment.GetEnvironmentVariable("AzureWebJobsStorage"));
var tableClient = storageAccount.CreateCloudTableClient();
var table = tableClient.GetTableReference("day032019");
await table.CreateIfNotExistsAsync();
var insertBatch = new TableBatchOperation();
foreach (var pngImage in pngImages)
{
var pngImageFileName = pngImage.Name.Substring(pngImage.Name.LastIndexOf("/") + 1);
var entity = new PngImageEntity(pngImageFileName)
{
Url = pngImage.Url
};
insertBatch.Add(TableOperation.Insert(entity));
}
await table.ExecuteBatchAsync(insertBatch);
log.LogInformation($"Persisted {pngImages.Count()} PNG images.");
}
public void ClearTable()
{
var getPersonOperation = TableOperation.Retrieve <PersonEntity>(LastName, FirstName);
var tableResult = cloudTable.Execute(getPersonOperation);
var person = (PersonEntity)tableResult.Result;
if (person != null)
{
Assert.AreEqual(LastName, person.LastName());
cloudTable.Execute(TableOperation.Delete(person));
}
TableQuery <AnimalEntity> animalsQuery = new TableQuery <AnimalEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, AnimalEntity.AnimalPartition));
TableBatchOperation tableBatchOperation = new TableBatchOperation();
foreach (AnimalEntity animalEntity in cloudTable.ExecuteQuery(animalsQuery))
{
var tableOperation = TableOperation.Delete(animalEntity);
tableBatchOperation.Add(tableOperation);
}
if (tableBatchOperation.Any())
{
cloudTable.ExecuteBatch(tableBatchOperation);
}
}
protected override async Task RemoveKeysAsync(IEnumerable <string> rowKeys, string partitionKey = null)
{
partitionKey = partitionKey ?? _defaultPartitionKey;
ArgCheck.NotNull(nameof(partitionKey), partitionKey);
var batch = new TableBatchOperation();
foreach (var rowKey in rowKeys)
{
batch.Add(TableOperation.Delete(new DynamicTableEntity(partitionKey, rowKey)
{
ETag = "*"
}));
if (batch.Count == 100)
{
await _table.ExecuteBatchAsync(batch).ConfigureAwait(false);
batch = new TableBatchOperation();
}
}
if (batch.Count > 0)
{
await _table.ExecuteBatchAsync(batch).ConfigureAwait(false);
}
}
public async Task ProcessAsync(CancellationToken token)
{
var queue = await InitializeQueue();
var table = await InitializeTable();
while (true)
{
token.ThrowIfCancellationRequested();
var messages = await queue.GetMessagesAsync(10, token);
if (!messages.Any())
{
await Task.Delay(TimeSpan.FromMinutes(1), token);
continue;
}
var batch = new TableBatchOperation();
foreach (var message in messages)
{
var order = JsonConvert.DeserializeObject <Order>(message.AsString);
order.PartitionKey = Guid.NewGuid().ToString();
batch.Add(TableOperation.Insert(order));
}
await table.ExecuteBatchAsync(batch, token);
}
}
public async Task StoreCategoryLinks(CategoryGroup categoryGroup, string categoryName, IEnumerable <CategoryLinkChange> changes, CancellationToken cancellationToken)
{
Ensure.String.IsNotNullOrWhiteSpace(categoryName, nameof(categoryName));
Ensure.Any.IsNotNull(changes, nameof(changes));
var table = GetTable(TableName);
var batch = new TableBatchOperation();
foreach (var change in changes)
{
if (batch.Count == 100)
{
// Batches can only handle 100 items, need to execute this batch
await ExecuteBatch(table, batch, cancellationToken).ConfigureAwait(false);
batch.Clear();
}
var operation = BuildLinkChangeTableOperation(categoryGroup, categoryName, change);
batch.Add(operation);
}
if (batch.Count == 0)
{
// We were provided a changes instance but no changes to be made
return;
}
await ExecuteBatch(table, batch, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// method to delete multiple entities from azure table
/// </summary>
/// <typeparam name="T">Takes table entity type as input</typeparam>
/// <param name="tablename">takes table name as input</param>
/// <param name="entitieslist">takes entities list as input</param>
public static void RemoveEntities <T>(string tablename, List <T> entitieslist) where T : ITableEntity, new()
{
//Get Caller Method name
string callerMethodName = string.Empty;
try
{
//Get Caller Method name from CallerInformation class
callerMethodName = CallerInformation.TrackCallerMethodName();
//get's azure table instance
CloudTable UserBackendConfigurationTable = GetAzureTableInstance(tablename);
TableBatchOperation batchOperation = new TableBatchOperation();
//insert list of entities into batch operation
foreach (T entity in entitieslist)
{
batchOperation.Add(TableOperation.Delete(entity));
}
UserBackendConfigurationTable.ExecuteBatch(batchOperation);
}
catch (Exception exception)
{
//write exception into application insights
InsightLogger.Exception(exception.Message + " - Error in DataProver while removing entity from " + tablename, exception, callerMethodName);
throw new Exception();
}
}
public async Task DeleteResponsesAsync(string id)
{
var query = new TableQuery <QuizResponseEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, id));
var token = new TableContinuationToken();
do
{
var result = await _quizResponsesTable.ExecuteQuerySegmentedAsync(query, token);
var tableBatchOperation = new TableBatchOperation();
foreach (var entity in result)
{
var deleteOperation = TableOperation.Delete(entity);
tableBatchOperation.Add(deleteOperation);
if (tableBatchOperation.Count == 100)
{
// Single batch can contain max. 100 items.
await _quizResponsesTable.ExecuteBatchAsync(tableBatchOperation);
tableBatchOperation = new TableBatchOperation();
}
}
if (tableBatchOperation.Any())
{
await _quizResponsesTable.ExecuteBatchAsync(tableBatchOperation);
}
token = result.ContinuationToken;
} while (token != null);
}
public async Task <IList <TableResult> > InsertBatchAsync(IEnumerable <UserMedia> entities, CancellationToken ct = default(CancellationToken))
{
var tasks = new List <IList <TableResult> >();
var entityBatches = entities.GroupAndSlice <UserMedia, string>(
_maxBatchEntityCount,
um => um.PartitionKey,
KeyGroupPredicate
);
foreach (var entityBatch in entityBatches)
{
var tbo = new TableBatchOperation();
foreach (var entity in entityBatch)
{
tbo.Add(TableOperation.Insert(entity));
}
ICancellableAsyncResult ar = _cloudTable.BeginExecuteBatch(tbo, null, null);
ct.Register(ar.Cancel);
var batchTask = await Task.Factory.FromAsync <IList <TableResult> >(ar, _cloudTable.EndExecuteBatch).ConfigureAwait(false);
tasks.Add(batchTask);
}
return(tasks.SelectMany(t => t).ToList());
}
public void TableBatchAddNullShouldThrow()
{
TableBatchOperation batch = new TableBatchOperation();
try
{
batch.Add(null);
Assert.Fail();
}
catch (ArgumentNullException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
try
{
batch.Insert(0, null);
Assert.Fail();
}
catch (ArgumentNullException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
}
/// <summary>
/// Emit a batch of log events, running to completion synchronously.
/// </summary>
/// <param name="events">The events to emit.</param>
/// <remarks>Override either <see cref="PeriodicBatchingSink.EmitBatch"/> or <see cref="PeriodicBatchingSink.EmitBatchAsync"/>,
/// not both.</remarks>
protected override void EmitBatch(IEnumerable <LogEvent> events)
{
var operation = new TableBatchOperation();
var first = true;
foreach (var logEvent in events)
{
if (first)
{
//check to make sure the partition key is not the same as the previous batch
if (partitionKey != logEvent.Timestamp.Ticks)
{
batchRowId = 0; //the partitionkey has been reset
partitionKey = logEvent.Timestamp.Ticks; //store the new partition key
}
first = false;
}
var logEventEntity = new LogEventEntity(logEvent, _formatProvider, partitionKey);
logEventEntity.RowKey += "|" + batchRowId;
operation.Add(TableOperation.Insert(logEventEntity));
batchRowId++;
}
_table.ExecuteBatch(operation);
}
/// <summary>
/// Inserts a list of table entries as a batch.
/// </summary>
/// <typeparam name="T">DTO that inherits from TableEntity</typeparam>
/// <param name="data">The data.</param>
public void InsertBatch <T>(IEnumerable <T> data) where T : TableEntity
{
// Create the batch operation.
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (var entityGroup in data.GroupBy(f => f.PartitionKey))
{
// Add both customer entities to the batch insert operation.
foreach (TableEntity entity in entityGroup)
{
if (batchOperation.Count < 100)
{
batchOperation.Add(TableOperation.InsertOrReplace(entity));
}
else
{
// Execute the batch operation.
table.ExecuteBatch(batchOperation);
batchOperation = new TableBatchOperation {
TableOperation.InsertOrReplace(entity)
};
}
}
table.ExecuteBatch(batchOperation);
batchOperation = new TableBatchOperation();
}
if (batchOperation.Count > 0)
{
table.ExecuteBatch(batchOperation);
}
}
// POST api/values
public string Post([FromBody] List <Data> values)
{
try
{
// 構成ファイルから Azure Storage への接続文字列を取得
string setting = CloudConfigurationManager.GetSetting("StorageConnectionString");
CloudStorageAccount account = CloudStorageAccount.Parse(setting);
// Create batch.
TableBatchOperation batch = new TableBatchOperation();
foreach (var value in values)
{
// Add Insert operation to batch.
batch.Add(TableOperation.Insert(new EventEntity(value)));
}
// Create the table client.
CloudTableClient client = account.CreateCloudTableClient();
// Get the CloudTable object reference that represents the "event" table.
CloudTable table = client.GetTableReference("event");
table.CreateIfNotExists();
// Execute batch operation.
table.ExecuteBatch(batch);
return("OK");
}
catch (Exception e)
{
return(e.Message);
}
}
private static async Task DeleteByPartitionKey(CloudTable table, string partitionKey)
{
var filter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
var query = new TableQuery <TableEntity>().Where(filter);
TableContinuationToken token = null;
do
{
var queryResult = await table.ExecuteQuerySegmentedAsync(query, token);
token = queryResult.ContinuationToken;
var batches = queryResult.Batch(100);
foreach (var batch in batches)
{
var tableBatchOperation = new TableBatchOperation();
foreach (var entity in batch)
{
tableBatchOperation.Add(TableOperation.Delete(entity));
}
await table.ExecuteBatchAsync(tableBatchOperation);
}
} while (token != null);
}
public async Task Persist(ProjectionMetaData metadata)
{
Trace.TraceInformation("Preparing batch for projection {0}.", metadata.ProjectionType);
var batch = new TableBatchOperation();
var entity = new DictionaryTableEntity
{
PartitionKey = metadata.ProjectionType,
RowKey = Handler.Current()
};
entity.Add("ProjectionHash", metadata.ProjectionHash);
batch.Add(TableOperation.InsertOrReplace(entity));
Trace.TraceInformation("Executing batch for projection {0}.", metadata.ProjectionType);
await _table.ExecuteBatchAsync(batch).ContinueWith(r =>
{
Trace.TraceInformation("Batch for projection {0} complete {1} exceptions.", metadata.ProjectionType, r.Exception != null ? "with" : "without");
if (r.Exception != null)
{
r.Exception.Handle(exception =>
{
Trace.TraceError(exception.ToString());
return true;
});
}
});
}
/// <summary>
/// Execute a DML operation
/// </summary>
/// <typeparam name="T">Class type of the business object</typeparam>
/// <param name="batchOperation">TableBatchOperation to execute</param>
public void ExecuteNonQuery <T>(TableBatchOperation batchOperation)
where T : class
{
if (batchOperation.Count > 0)
{
TableBatchOperation batchPage = new TableBatchOperation();
// all entities in a batch must have the same partition key:
foreach (IEnumerable <TableOperation> operations in batchOperation.GroupBy(o => o.Entity.PartitionKey))
{
// order elements in a partition by row key so that we reduce tablescans
foreach (TableOperation operation in operations.OrderBy(o => o.Entity.RowKey))
{
batchPage.Add(operation);
if (batchPage.Count == 100)
{
_currentTableReference.ExecuteBatch(batchPage);
batchPage.Clear();
}
}
}
// get the remaining
if (batchPage.Count > 0)
{
_currentTableReference.ExecuteBatch(batchPage);
}
}
}
private static int Insert(string partitionKey, CloudTable itemTable, IEnumerable <Item> items)
{
int numOperations = 0;
var batch = new TableBatchOperation();
foreach (var item in items)
{
batch.Add(TableOperation.Insert(item));
if (batch.Count == 100)
{
itemTable.ExecuteBatch(batch);
batch.Clear();
numOperations++;
}
}
if (batch.Count > 0)
{
itemTable.ExecuteBatch(batch);
numOperations++;
}
return(numOperations);
}
/// <summary>
/// Mark the items as deleted in the active partition (this updates day-to-day users)
/// </summary>
/// <param name="activePartitionKey"></param>
/// <param name="itemTable"></param>
/// <param name="itemKeys"></param>
private static async void DeleteInActivePartition(string activePartitionKey, CloudTable itemTable, List <ItemToRemove> itemKeys)
{
var batch = new TableBatchOperation();
foreach (var itemKey in itemKeys)
{
batch.Add(TableOperation.InsertOrReplace(new DeletedItemV1 {
PartitionKey = activePartitionKey,
RowKey = Guid.NewGuid().ToString(),
ItemPartitionKey = itemKey.Partition,
ItemRowKey = itemKey.Id
}));
if (batch.Count == 100)
{
await itemTable.ExecuteBatchAsync(batch);
batch.Clear();
}
}
if (batch.Count > 0)
{
await itemTable.ExecuteBatchAsync(batch);
}
}
public void TableBatchAddNullShouldThrow()
{
TableBatchOperation batch = new TableBatchOperation();
try
{
batch.Add(null);
Assert.Fail();
}
catch (ArgumentNullException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
try
{
batch.Insert(0, null);
Assert.Fail();
}
catch (ArgumentNullException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
}
public async Task DeleteIfExistsAsync(CancellationToken cancellationToken)
{
if (!await this.table.ExistsAsync().ConfigureAwait(false))
{
return;
}
var query = new TableQuery <PartitionInfoEntity>().Where(TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, this.taskHubName));
TableContinuationToken continuationToken = null;
do
{
var batch = await this.table.ExecuteQuerySegmentedAsync <PartitionInfoEntity>(query, continuationToken, null, null, cancellationToken).ConfigureAwait(false);
if (batch.Count() > 0)
{
// delete all entities in this batch. Max partition number is 32 so it always fits.
TableBatchOperation tableBatch = new TableBatchOperation();
foreach (var e in batch)
{
tableBatch.Add(TableOperation.Delete(e));
}
await this.table.ExecuteBatchAsync(tableBatch).ConfigureAwait(false);
}
}while (continuationToken != null);
}
public async Task Add(TableOperation operation)
{
batchOperation.Add(operation);
if (operation.Entity == key)
{
batchContainsKey = true;
}
if (batchOperation.Count == AzureTableConstants.MaxBatchSize - (batchContainsKey ? 0 : 1))
{
// the key serves as a synchronizer, to prevent modification by multiple grains under edge conditions,
// like duplicate activations or deployments.Every batch write needs to include the key,
// even if the key values don't change.
if (!batchContainsKey)
{
if (string.IsNullOrEmpty(key.ETag))
{
batchOperation.Insert(key);
}
else
{
batchOperation.Replace(key);
}
}
await Flush().ConfigureAwait(false);
batchOperation.Clear();
batchContainsKey = false;
}
}
private void Index(List <ChainPartEntry> chainParts, CancellationToken cancellationToken = default(CancellationToken))
{
this.logger.LogTrace("()");
CloudTable table = this.Configuration.GetChainTable();
TableBatchOperation batch = new TableBatchOperation();
var last = chainParts[chainParts.Count - 1];
foreach (var entry in chainParts)
{
batch.Add(TableOperation.InsertOrReplace(entry.ToEntity()));
if (batch.Count == 100)
{
table.ExecuteBatchAsync(batch).GetAwaiter().GetResult();
batch = new TableBatchOperation();
}
IndexerTrace.RemainingBlockChain(entry.ChainOffset, last.ChainOffset + last.BlockHeaders.Count - 1);
}
if (batch.Count > 0)
{
this.logger.LogTrace("Batch count: {0}", batch.Count);
table.ExecuteBatchAsync(batch, null, null, cancellationToken).GetAwaiter().GetResult();
}
this.logger.LogTrace("(-)");
}
/// <summary>
/// Emit a batch of log events, running to completion synchronously.
/// </summary>
/// <param name="events">The events to emit.</param>
/// <remarks>Override either <see cref="PeriodicBatchingSink.EmitBatch"/> or <see cref="PeriodicBatchingSink.EmitBatchAsync"/>,
/// not both.</remarks>
protected override void EmitBatch(IEnumerable<LogEvent> events)
{
var operation = new TableBatchOperation();
var first = true;
foreach (var logEvent in events)
{
if (first)
{
//check to make sure the partition key is not the same as the previous batch
if (partitionKey != logEvent.Timestamp.Ticks)
{
batchRowId = 0; //the partitionkey has been reset
partitionKey = logEvent.Timestamp.Ticks; //store the new partition key
}
first = false;
}
var logEventEntity = new LogEventEntity(logEvent, _formatProvider, partitionKey);
logEventEntity.RowKey += "|" + batchRowId;
operation.Add(TableOperation.Insert(logEventEntity));
batchRowId++;
}
_table.ExecuteBatch(operation);
}
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> messages)
{
var batch = new TableBatchOperation();
foreach (var msg in messages)
{
var snap = await JsonConvert.DeserializeObjectAsync <BusSnapshotInfo>(Encoding.UTF8.GetString(msg.GetBytes()));
var entity = new DynamicTableEntity(snap.RouteShortName, snap.VehicleId.ToString());
entity.Properties.Add("RouteShortName", EntityProperty.GeneratePropertyForString(snap.RouteShortName));
entity.Properties.Add("VehicleId", EntityProperty.GeneratePropertyForInt(snap.VehicleId));
entity.Properties.Add("TripId", EntityProperty.GeneratePropertyForInt(snap.TripId));
entity.Properties.Add("Latitude", EntityProperty.GeneratePropertyForDouble(snap.Latitude));
entity.Properties.Add("Longitude", EntityProperty.GeneratePropertyForDouble(snap.Longitude));
entity.Properties.Add("DirectionOfTravel", EntityProperty.GeneratePropertyForString(snap.DirectionOfTravel.ToString()));
entity.Properties.Add("NextStopId", EntityProperty.GeneratePropertyForInt(snap.NextStopId));
entity.Properties.Add("Timeliness", EntityProperty.GeneratePropertyForString(snap.Timeliness.ToString()));
entity.Properties.Add("TimelinessOffset", EntityProperty.GeneratePropertyForInt(snap.TimelinessOffset));
entity.Properties.Add("Timestamp", EntityProperty.GeneratePropertyForDateTimeOffset(snap.Timestamp));
batch.Add(TableOperation.InsertOrReplace(entity));
}
var tableClient = _account.CreateCloudTableClient();
var table = tableClient.GetTableReference("snapshots");
await table.CreateIfNotExistsAsync();
await table.ExecuteBatchAsync(batch);
await context.CheckpointAsync();
}
void ClearTable(CloudTable table)
{
foreach (var partitionKey in this.PartitionKeys())
{
TableBatchOperation batchDelete = new TableBatchOperation();
// gets all the entities in the table for this partition key
string partitionCondition = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
List <DynamicTableEntity> entities = table.ExecuteQuery(new TableQuery().Where(partitionCondition)).ToList();
entities.ForEach(e =>
{
batchDelete.Add(TableOperation.Delete(e));
// Azure has a limit on batch operations
if (batchDelete.Count == 100)
{
table.ExecuteBatch(batchDelete);
batchDelete = new TableBatchOperation();
}
});
// flush out whatever is left
if (batchDelete.Count > 0)
{
table.ExecuteBatch(batchDelete);
}
}
}
private async Task EmptyTableAsync(CloudTable table)
{
var results = await _repository.ExecuteQuerySafeAsync(table, new TableQuery());
if (results.Any())
{
TableBatchOperation batch = new TableBatchOperation();
foreach (var entity in results)
{
batch.Add(TableOperation.Delete(entity));
if (batch.Count == 100)
{
var result = await table.ExecuteBatchAsync(batch);
batch = new TableBatchOperation();
}
}
if (batch.Count > 0)
{
await table.ExecuteBatchAsync(batch);
}
}
}
/// <summary>
/// Inserts the update batch terrain data.
/// </summary>
/// <param name="tableName">Name of the table.</param>
/// <param name="batchData">The batch data.</param>
public void InsertUpdateBatchPMSEUnscheduledAdjustment(string tableName, List <TableOperation> batchData)
{
var cloudAccount = this.GetCloudStorageAccount();
var cloudTable = cloudAccount.CreateCloudTableClient().GetTableReference(tableName);
cloudTable.CreateIfNotExists();
TableBatchOperation batchOperation = new TableBatchOperation();
for (int i = 0; i < batchData.Count; i += 100)
{
batchOperation.Clear();
for (int j = i; j < i + 100; j++)
{
if (j >= batchData.Count)
{
break;
}
batchOperation.Add(batchData[j]);
}
cloudTable.ExecuteBatch(batchOperation);
}
}
private static int Delete(string partitionKey, CloudTable itemTable, List <string> itemKeys)
{
int numOperations = 0;
var batch = new TableBatchOperation();
foreach (var itemKey in itemKeys)
{
var entity = new DynamicTableEntity(partitionKey, itemKey);
entity.ETag = "*";
entity.Properties.Add("IsActive", new EntityProperty(false));
batch.Add(TableOperation.Merge(entity));
if (batch.Count == 100)
{
itemTable.ExecuteBatch(batch);
batch.Clear();
numOperations++;
}
}
if (batch.Count > 0)
{
itemTable.ExecuteBatch(batch);
numOperations++;
}
return(numOperations);
}
public async Task Save(Page[] pages)
{
var table = await GetTable();
var batchInsertOperation = new TableBatchOperation();
foreach (var page in pages)
{
batchInsertOperation.Add(TableOperation.Insert(ToPageTableEntity(page)));
}
await table.ExecuteBatchAsync(batchInsertOperation);
}
public void Delete()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference(tableName);
var entries = table.CreateQuery<EventTableEntity>().Where(e => e.PartitionKey == Id).ToArray();
var delete = new TableBatchOperation();
foreach (var eventTableEntity in entries)
{
delete.Add(TableOperation.Delete(eventTableEntity));
}
table.ExecuteBatch(delete);
}
public void AddVotes(IEnumerable<Vote> votes)
{
CreateTableStorageIfDoesntExist();
var tableClient = GetTableClient();
var table = tableClient.GetTableReference(_tableName);
var batch = new TableBatchOperation();
foreach (var vote in votes)
{
batch.Add(TableOperation.InsertOrReplace(vote));
}
table.ExecuteBatch(batch);
}
public async Task<IEnumerable<Response<JsonObject>>> BatchPostAsync(IEnumerable<JsonObject> jsonObjects)
{
if (jsonObjects == null)
throw new ArgumentNullException("jsonObjects");
var table = await DefineTableAsync(CreateCloudTable).ConfigureAwait(false);
var batch = new TableBatchOperation();
foreach (var entity in jsonObjects.Select(json => json.ToDynamicEntity()))
batch.Add(TableOperation.Insert(entity));
var results = await table.ExecuteBatchAsync(batch).ConfigureAwait(false);
return results
.Select(result => result.Result as DynamicTableEntity)
.Select(entity => entity.ToJsonObject())
.Select(jsonObj => new Response<JsonObject>(HttpStatusCode.Created, jsonObj));
}
public async Task Persist(string streamtype, string id, IEnumerable<ISourcedEvent> pendingEvents)
{
Trace.TraceInformation("Preparing batch for stream {0}.", streamtype);
var batch = new TableBatchOperation();
foreach (var @event in pendingEvents)
{
var entity = new DictionaryTableEntity
{
PartitionKey = streamtype,
RowKey = id + "_" + @event.Version.ToString(VersionKeyFormat)
};
var compressed = false;
var body = Json.Encode(@event);
if (body.Length > 32*1024)
{
body = new GzipCompression().Compress(body);
compressed = true;
}
entity.Add("EventType", @event.GetType().FullName);
entity.Add("SourceId", @event.SourceId);
entity.Add("Version", @event.Version);
entity.Add("Body", body);
entity.Add("Compressed", compressed);
batch.Add(TableOperation.Insert(entity));
}
Trace.TraceInformation("Executing batch on stream {0}.", streamtype);
await _table.ExecuteBatchAsync(batch).ContinueWith(r =>
{
Trace.TraceInformation("Batch on stream {0} complete {1} exceptions.", streamtype, r.Exception != null ? "with" : "without");
if (r.Exception != null)
{
r.Exception.Handle(exception =>
{
Trace.TraceError(exception.ToString());
return true;
});
}
});
}
public void SaveChanges()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference(tableName);
var entries = newEvents.Select(e => new EventTableEntity()
{
PartitionKey = id,
RowKey = e.Value.Index.ToString().PadLeft(8, '0'),
Content = e.Value.Content
});
var insert = new TableBatchOperation();
foreach (var eventTableEntity in entries)
{
insert.Add(TableOperation.InsertOrReplace(eventTableEntity));
}
table.ExecuteBatch(insert);
newEvents.Clear();
}
private void PersistStoriesAsync(IList<IStory> stories)
{
var tableBatchOperation = new TableBatchOperation();
foreach (var story in stories)
{
tableBatchOperation.Add(TableOperation.Insert(StoryTableEntity.ToStoryTableEntity(story)));
}
try
{
Retrier.Retry<object>(() =>
{
this.storiesTable.ExecuteBatch(tableBatchOperation);
return null;
});
}
catch (StorageException)
{
}
}
public void ClearTable()
{
var getPersonOperation = TableOperation.Retrieve<PersonEntity>(LastName, FirstName);
var tableResult = cloudTable.Execute(getPersonOperation);
var person = (PersonEntity)tableResult.Result;
if (person != null)
{
Assert.AreEqual(LastName, person.LastName());
cloudTable.Execute(TableOperation.Delete(person));
}
TableQuery<AnimalEntity> animalsQuery = new TableQuery<AnimalEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, AnimalEntity.AnimalPartition));
TableBatchOperation tableBatchOperation = new TableBatchOperation();
foreach (AnimalEntity animalEntity in cloudTable.ExecuteQuery(animalsQuery))
{
var tableOperation = TableOperation.Delete(animalEntity);
tableBatchOperation.Add(tableOperation);
}
if (tableBatchOperation.Any())
{
cloudTable.ExecuteBatch(tableBatchOperation);
}
}
/// <inheritdoc />
public async Task<long> ExecuteDeleteAllAsync(CloudTable table, string partitionKey, string filter)
{
if (table == null)
{
throw new ArgumentNullException(nameof(table));
}
if (partitionKey == null)
{
throw new ArgumentNullException(nameof(partitionKey));
}
// Build query for retrieving exiting entries. We only ask for PK and RK.
TableQuery query = new TableQuery()
{
FilterString = filter,
SelectColumns = new List<string> { PartitionKey, RowKey },
};
AddPartitionKeyConstraint(query, partitionKey);
try
{
long totalCount = 0;
TableContinuationToken continuationToken = null;
do
{
DynamicTableEntity[] webHooks = (await ExecuteQueryAsync(table, query)).ToArray();
if (webHooks.Length == 0)
{
break;
}
// Delete query results in max of 100-count batches
int totalSegmentCount = webHooks.Length;
int segmentCount = 0;
do
{
TableBatchOperation batch = new TableBatchOperation();
int batchCount = Math.Min(totalSegmentCount - segmentCount, MaxBatchSize);
for (int cnt = 0; cnt < batchCount; cnt++)
{
DynamicTableEntity entity = webHooks[segmentCount + cnt];
entity.ETag = "*";
TableOperation operation = TableOperation.Delete(entity);
batch.Add(operation);
}
await ExecuteBatchAsync(table, batch);
segmentCount += batchCount;
}
while (segmentCount < totalSegmentCount);
totalCount += segmentCount;
}
while (continuationToken != null);
return totalCount;
}
catch (Exception ex)
{
string errorMessage = GetStorageErrorMessage(ex);
int statusCode = GetStorageStatusCode(ex);
string msg = string.Format(CultureInfo.CurrentCulture, AzureStorageResources.StorageManager_OperationFailed, statusCode, errorMessage);
_logger.Error(msg, ex);
throw new InvalidOperationException(msg, ex);
}
}
private async Task SendEventsAsync(IEnumerable<EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
if (events == null)
{
return;
}
DateTime now = DateTime.UtcNow;
string partitionKey = now.ToString("yyyyMMddhhmm") + KeySegmentSeparator + (transmissionSequenceNumber%MaxConcurrentPartitions).ToString("D2");
string rowKeyPrefix = now.ToString("ssfff");
try
{
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (EventData eventData in events)
{
DynamicTableEntity entity = this.ToTableEntity(eventData, partitionKey, rowKeyPrefix);
TableOperation insertOperation = TableOperation.Insert(entity);
batchOperation.Add(insertOperation);
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
// CONSIDER exposing TableRequestOptions and OperationContext for the batch operation
await this.cloudTable.ExecuteBatchAsync(batchOperation, null, null, cancellationToken);
this.ReportListenerHealthy();
}
catch (Exception e)
{
this.ReportListenerProblem("Diagnostics data upload has failed." + Environment.NewLine + e.ToString());
}
}
public void TableBatchLockToPartitionKey()
{
TableBatchOperation batch = new TableBatchOperation();
batch.Add(TableOperation.Insert(GenerateRandomEntity("foo")));
try
{
batch.Add(TableOperation.Insert(GenerateRandomEntity("foo2")));
Assert.Fail();
}
catch (ArgumentException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
// should reset pk lock
batch.RemoveAt(0);
batch.Add(TableOperation.Insert(GenerateRandomEntity("foo2")));
try
{
batch.Add(TableOperation.Insert(GenerateRandomEntity("foo2")));
}
catch (ArgumentException)
{
Assert.Fail();
}
catch (Exception)
{
Assert.Fail();
}
}
public void TableBatchAddQueryAndOneMoreOperationShouldThrow()
{
TableBatchOperation batch = new TableBatchOperation();
try
{
batch.Add(TableOperation.Retrieve("foo", "bar"));
batch.Add(TableOperation.Insert(GenerateRandomEntity("foo")));
Assert.Fail();
}
catch (ArgumentException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
batch.Clear();
try
{
batch.Add(TableOperation.Insert(GenerateRandomEntity("foo")));
batch.Add(TableOperation.Retrieve("foo", "bar"));
Assert.Fail();
}
catch (ArgumentException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
batch.Clear();
try
{
batch.Add(TableOperation.Retrieve("foo", "bar"));
batch.Insert(0, TableOperation.Insert(GenerateRandomEntity("foo")));
Assert.Fail();
}
catch (ArgumentException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
try
{
batch.Insert(0, TableOperation.Insert(GenerateRandomEntity("foo")));
batch.Insert(0, TableOperation.Retrieve("foo", "bar"));
Assert.Fail();
}
catch (ArgumentException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
}
public async Task<bool> RemoveSubscriptions(IEnumerable<UserSubscription> subscriptionsToRemove)
{
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable userSubscriptionsTable = tableClient.GetTableReference("userSubscriptions");
var tableExists = await userSubscriptionsTable.ExistsAsync();
if (!tableExists)
{
return false;
}
List<UserSubscriptionEntity> activeUserSubscriptionEntities = new List<UserSubscriptionEntity>();
Expression<Func<UserSubscriptionEntity, bool>> filter =
(x) => x.PartitionKey == subscriptionsToRemove.First().UserId.ToString();
Action<IEnumerable<UserSubscriptionEntity>> processor = activeUserSubscriptionEntities.AddRange;
await ObtainUserSubscriptionEntities(userSubscriptionsTable, filter, processor);
TableBatchOperation deletionBatchOperation = new TableBatchOperation();
foreach (var userSubscription in subscriptionsToRemove)
{
var entity = activeUserSubscriptionEntities.SingleOrDefault(
x => x.PartitionKey == userSubscription.UserId.ToString() &&
x.RowKey == userSubscription.FriendId.ToString());
if (entity != null)
{
deletionBatchOperation.Add(TableOperation.Delete(entity));
}
}
if (deletionBatchOperation.Any())
{
await userSubscriptionsTable.ExecuteBatchAsync(deletionBatchOperation);
}
return true;
}
/// <summary>
/// Clears the journal up to and including, the provided <paramref name="id"/>.
/// </summary>
/// <param name="id">
/// The id to clear up to.
/// </param>
/// <returns>
/// A <see cref="Task"/> representing the work performed.
/// </returns>
public async Task Clear(long id)
{
// Azure Table Service supports batches of 100, so we chop the results up and batch the deletions.
await this.BatchReadFrom(
id,
async entities =>
{
foreach (var batch in entities.Batch(100))
{
var deletion = new TableBatchOperation();
foreach (var entity in batch)
{
deletion.Add(TableOperation.Delete(new EventEntity(this.partitionKey, entity.Id)));
}
await this.store.ExecuteBatchAsync(deletion);
}
},
CancellationToken.None);
}
public void SendDifferentEntityToTable()
{
TableBatchOperation batchOperation = new TableBatchOperation();
var dog = new AnimalEntity("Lassie")
{
Color = "Mixed"
};
var insertOperation = TableOperation.InsertOrReplace(dog);
batchOperation.Add(insertOperation);
dog = new AnimalEntity("Pluto")
{
Color = "Mixed"
};
insertOperation = TableOperation.InsertOrMerge(dog);
batchOperation.Add(insertOperation);
cloudTable.ExecuteBatch(batchOperation);
}
private List<Exception> SaveConcurrent(int numberOfCalls, bool useBatch)
{
// Do several calls to force some timeouts
// or manually disconnect the network while this test is running.
string partitionKey = Guid.NewGuid().ToString();
var exceptions = new ConcurrentBag<Exception>();
var entities = Enumerable.Range(0, numberOfCalls)
.Select(x => new TestTableEntity { PartitionKey = partitionKey, RowKey = x.ToString() })
.ToList();
var barrier = new Barrier(numberOfCalls);
var countdown = new CountdownEvent(numberOfCalls);
foreach (var entity in entities)
{
ThreadPool.QueueUserWorkItem(x =>
{
try
{
var client = account.CreateCloudTableClient();
client.RetryPolicy = new NoRetry();
// Explicitly set a VERY short timeout, to force a timeout very frequently.
client.ServerTimeout = TimeSpan.FromSeconds(1);
client.MaximumExecutionTime = TimeSpan.FromSeconds(1);
var table = client.GetTableReference(tableName);
// Create the TableOperation that inserts the customer entity.
var insertOperation = TableOperation.Insert((TestTableEntity)x);
barrier.SignalAndWait();
// Execute the insert operation.
if (useBatch)
{
var batch = new TableBatchOperation();
batch.Add(insertOperation);
table.ExecuteBatch(batch);
}
else
{
table.Execute(insertOperation);
}
}
catch (Exception ex)
{
exceptions.Add(ex);
}
finally
{
countdown.Signal();
}
}, entity);
}
countdown.Wait();
if (exceptions.Count == 0)
{
Assert.Inconclusive("No exceptions were thrown to check if they are transient");
}
return exceptions.ToList();
}
public void TableBatchAddQueryAndOneMoreOperationShouldThrow()
{
TableBatchOperation batch = new TableBatchOperation();
TableOperation operation = TableOperation.Retrieve<DynamicReplicatedTableEntity>("foo", "bar");
try
{
batch.Add(operation);
Assert.IsTrue(batch.Contains(operation));
batch.Add(TableOperation.Insert(GenerateRandomEnitity("foo")));
Assert.Fail();
}
catch (ArgumentException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
batch.Clear();
Assert.IsFalse(batch.Contains(operation));
try
{
batch.Add(TableOperation.Insert(GenerateRandomEnitity("foo")));
batch.Add(TableOperation.Retrieve<DynamicReplicatedTableEntity>("foo", "bar"));
Assert.Fail();
}
catch (ArgumentException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
batch.Clear();
try
{
batch.Add(TableOperation.Retrieve<DynamicReplicatedTableEntity>("foo", "bar"));
batch.Insert(0, TableOperation.Insert(GenerateRandomEnitity("foo")));
Assert.Fail();
}
catch (ArgumentException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
try
{
batch.Insert(0, TableOperation.Insert(GenerateRandomEnitity("foo")));
batch.Insert(0, TableOperation.Retrieve<DynamicReplicatedTableEntity>("foo", "bar"));
Assert.Fail();
}
catch (ArgumentException)
{
// no op
}
catch (Exception)
{
Assert.Fail();
}
}
public int Delete()
{
ValidFileExist();
var operations = new TableBatchOperation();
if (IsDirectory())
{
foreach (var i in FileTable.ExecuteQuery(PrefixQuery()))
{
if (i.Type != FileEntity.Directory) new BlobFile2(i).GetBlob().Delete();
operations.Add(TableOperation.Delete(i));
}
}
else
{
GetBlob().Delete();
}
operations.Add(TableOperation.Delete(_entity));
FileTable.ExecuteBatch(operations);
return operations.Count;
}
public int Move(string p)
{
if (_path.Contains(p)) throw new Exception("You can't move a directory into itself");
var target = new BlobFile2(_username, p);
if (target.Exists())
{
if (target.IsDirectory())
{
p += "/" + Path().Name();
}
else
{
target.ValidateFileNotExist();
}
}
new BlobFile2(_username, p).ValidateFileNotExist();
var operations = new TableBatchOperation();
var path = new FilePath(p);
if (IsDirectory())
{
foreach (FileEntity i in FileTable.ExecuteQuery(PrefixQuery()))
{
i.Path = i.Path.Replace(_path.Path(), path.Path());
i.Parent = i.Parent.Replace(_path.Path(), path.Path());
operations.Add(TableOperation.Replace(i));
}
}
_entity.Path = path.Path();
_entity.Parent = path.BasePath();
operations.Add(TableOperation.Replace(_entity));
this._path = path;
FileTable.ExecuteBatch(operations);
return operations.Count;
}
public int Copy(string p)
{
if (_path.Contains(p)) throw new Exception("You can't copy a directory into itself");
var target = new BlobFile2(_username, p);
if (target.Exists())
{
if (target.IsDirectory())
{
p += "/" + Path().Name();
}
else
{
target.ValidateFileNotExist();
}
}
new BlobFile2(_username, p).ValidateFileNotExist();
var operations = new TableBatchOperation();
var path = new FilePath(p);
if (IsDirectory())
{
foreach (FileEntity i in FileTable.ExecuteQuery(PrefixQuery()))
{
var n = new FileEntity()
{
ContentType = i.ContentType,
Key = Guid.NewGuid().ToString("N"),
LastModified = DateTime.UtcNow,
Length = i.Length,
Path = i.Path.Replace(_path.Path(), path.Path()),
PartitionKey = i.PartitionKey,
Parent = i.Parent.Replace(_path.Path(), path.Path()),
RowKey = DateTime.UtcNow.Ticks.ToString("D") + _rnd.Next(1000, 9999).ToString("D"),
Type = i.Type
};
operations.Add(TableOperation.Insert(n));
if (i.Type != FileEntity.Directory)
_container.GetBlockBlobReference(n.Key)
.StartCopyFromBlob(_container.GetBlockBlobReference(i.Key));
}
}
var m = new FileEntity()
{
ContentType = _entity.ContentType,
Key = Guid.NewGuid().ToString("D"),
LastModified = DateTime.UtcNow,
Length = _entity.Length,
Path = path.Path(),
PartitionKey = _entity.PartitionKey,
Parent = path.BasePath(),
RowKey = DateTime.UtcNow.Ticks.ToString("N") + _rnd.Next(1000, 9999).ToString("D"),
Type = _entity.Type
};
if (!IsDirectory())
{
_container.GetBlockBlobReference(m.Key).StartCopyFromBlob(GetBlob());
}
operations.Add(TableOperation.Insert(m));
FileTable.ExecuteBatch(operations);
return operations.Count;
}
private void DoTableBatchBasicOperationsCheck(TablePayloadFormat format)
{
tableClient.DefaultRequestOptions.PayloadFormat = format;
string pk = Guid.NewGuid().ToString();
TableBatchOperation batch = new TableBatchOperation();
// Add insert
DynamicTableEntity ent1 = GenerateRandomEntity(pk);
TableOperation operation1 = TableOperation.Insert(ent1);
batch.Add(operation1);
DynamicTableEntity ent2 = GenerateRandomEntity(pk);
TableOperation operation2 = TableOperation.Insert(ent2);
batch.Add(operation2);
TableOperation[] operationsArray = new TableOperation[2];
batch.CopyTo(operationsArray, 0);
Assert.AreEqual(operation1.Entity.RowKey, operationsArray[0].Entity.RowKey);
Assert.AreEqual(operation1.OperationType, operationsArray[0].OperationType);
Assert.AreEqual(operation2.Entity.RowKey, operationsArray[1].Entity.RowKey);
Assert.AreEqual(operation2.OperationType, operationsArray[1].OperationType);
Assert.AreEqual(0, batch.IndexOf(operation1));
Assert.AreEqual(1, batch.IndexOf(operation2));
IEnumerator<TableOperation> enumerator = batch.GetEnumerator();
int totalCount = 0;
while (enumerator.MoveNext())
totalCount++;
Assert.AreEqual(2, totalCount);
Assert.IsTrue(batch.Remove(operation2));
Assert.IsFalse(batch.IsReadOnly);
TestHelper.ExpectedException<NotSupportedException>(() => batch[0] = operation1,
"Setter is not supported for TableBatchOperation");
}
