public async Task AddVersion(OwnershipRegistration registration, OwnershipOwner owner, string version)
{
CloudTableClient client = _account.CreateCloudTableClient();
CloudTable table = client.GetTableReference(OwnershipTableName);
TableBatchOperation batch = new TableBatchOperation();
batch.InsertOrReplace(new TypedEntity(registration.GetKey(), owner.GetKey(), OwnerType));
batch.InsertOrReplace(new TypedEntity(registration.GetKey(), version, PackageType));
await table.ExecuteBatchAsync(batch);
}
public TestFixture()
{
_config = new HttpConfiguration();
_config.Formatters.Add(new PlaintextMediaTypeFormatter());
_settingsManager = ScriptSettingsManager.Instance;
HostSettings = new WebHostSettings
{
IsSelfHost = true,
ScriptPath = Path.Combine(Environment.CurrentDirectory, @"..\..\..\..\sample"),
LogPath = Path.Combine(Path.GetTempPath(), @"Functions"),
SecretsPath = Path.Combine(Environment.CurrentDirectory, @"..\..\..\..\src\WebJobs.Script.WebHost\App_Data\Secrets")
};
WebApiConfig.Register(_config, _settingsManager, HostSettings);
HttpServer = new HttpServer(_config);
this.HttpClient = new HttpClient(HttpServer);
this.HttpClient.BaseAddress = new Uri("https://localhost/");
string connectionString = AmbientConnectionStringProvider.Instance.GetConnectionString("Storage");
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
BlobClient = storageAccount.CreateCloudBlobClient();
QueueClient = storageAccount.CreateCloudQueueClient();
TableClient = storageAccount.CreateCloudTableClient();
var table = TableClient.GetTableReference("samples");
table.CreateIfNotExists();
var batch = new TableBatchOperation();
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "1", Title = "Test Entity 1", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "2", Title = "Test Entity 2", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "3", Title = "Test Entity 3", Status = 1
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "4", Title = "Test Entity 4", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "5", Title = "Test Entity 5", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "6", Title = "Test Entity 6", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "7", Title = "Test Entity 7", Status = 0
});
table.ExecuteBatch(batch);
connectionString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.ServiceBus);
NamespaceManager = NamespaceManager.CreateFromConnectionString(connectionString);
WaitForHost();
}
// public NamespaceManager NamespaceManager { get; set; }
protected override async Task CreateTestStorageEntities()
{
// Don't call base.
var table = TableClient.GetTableReference("samples");
await table.CreateIfNotExistsAsync();
var batch = new TableBatchOperation();
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "1", Title = "Test Entity 1", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "2", Title = "Test Entity 2", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "3", Title = "Test Entity 3", Status = 1
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "4", Title = "Test Entity 4", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "5", Title = "Test Entity 5", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "6", Title = "Test Entity 6", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "7", Title = "Test Entity 7", Status = 0
});
await table.ExecuteBatchAsync(batch);
// TODO: This currently throws
// string connectionString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.ServiceBus);
// NamespaceManager = NamespaceManager.CreateFromConnectionString(connectionString);
}
protected override async Task CreateTestStorageEntities()
{
// Don't call base.
var table = TableClient.GetTableReference("samples");
await table.CreateIfNotExistsAsync();
var batch = new TableBatchOperation();
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "1", Title = "Test Entity 1", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "2", Title = "Test Entity 2", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "3", Title = "Test Entity 3", Status = 1
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "4", Title = "Test Entity 4", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "5", Title = "Test Entity 5", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "6", Title = "Test Entity 6", Status = 0
});
batch.InsertOrReplace(new TestEntity {
PartitionKey = "samples-python", RowKey = "7", Title = "Test Entity 7", Status = 0
});
await table.ExecuteBatchAsync(batch);
}
private void InitBank()
{
TableBatchOperation batchOperation = new TableBatchOperation();
Bank a1 = new Bank("1", 20000);
Bank a2 = new Bank("2", 1600);
Bank a3 = new Bank("3", 6000);
Bank a4 = new Bank("4", 9000);
Bank a5 = new Bank("5", 5000);
batchOperation.InsertOrReplace(a1);
batchOperation.InsertOrReplace(a2);
batchOperation.InsertOrReplace(a3);
batchOperation.InsertOrReplace(a4);
batchOperation.InsertOrReplace(a5);
_tableBank.ExecuteBatch(batchOperation);
}
private void InitBookstore()
{
TableBatchOperation batchOperation = new TableBatchOperation();
Bookstore a1 = new Bookstore("1", 5, 800);
Bookstore a2 = new Bookstore("2", 5, 900);
Bookstore a3 = new Bookstore("3", 5, 1000);
Bookstore a4 = new Bookstore("4", 5, 1100);
Bookstore a5 = new Bookstore("5", 5, 1500);
batchOperation.InsertOrReplace(a1);
batchOperation.InsertOrReplace(a2);
batchOperation.InsertOrReplace(a3);
batchOperation.InsertOrReplace(a4);
batchOperation.InsertOrReplace(a5);
_tableBookstore.ExecuteBatch(batchOperation);
}
private void InitStudents()
{
TableBatchOperation batchOperation = new TableBatchOperation();
Student a1 = new Student(1, "Ime1", "Prezime1");
Student a2 = new Student(2, "Ime2", "Prezime2");
Student a3 = new Student(3, "Ime3", "Prezime3");
Student a4 = new Student(4, "Ime4", "Prezime4");
Student a5 = new Student(5, "Ime5", "Prezime5");
batchOperation.InsertOrReplace(a1);
batchOperation.InsertOrReplace(a2);
batchOperation.InsertOrReplace(a3);
batchOperation.InsertOrReplace(a4);
batchOperation.InsertOrReplace(a5);
_table.ExecuteBatch(batchOperation);
}
public async static Task <int> ProcessEntities(DataTable dataTable, CloudTable table)
{
int batchCount = 0;
int totalRows = dataTable.Rows.Count - 1;
int returnTotalRows = totalRows;
_operations = new TableBatchOperation();
foreach (DataRow row in dataTable.Rows.Cast <DataRow>().Skip(1))
{
totalRows--;
batchCount++;
if (IsBatchCountExecutable(batchCount))
{
await ExecuteBatch(table);
batchCount = 0;
Console.WriteLine($"processed {_batchLimit} - " + Environment.NewLine + $"{totalRows}");
}
try
{
TableEntity entity = new TableEntity();
switch (table.Name)
{
case "Cities":
entity = GetCity(row);
break;
case "Schools":
entity = GetSchool(row);
break;
case "Statutes":
entity = GetStatute(row);
break;
case "Beats":
entity = GetBeat(row);
break;
default:
break;
}
DeduplicateBatch(entity);
_operations.InsertOrReplace(entity);
}
catch (Exception ex)
{
_log.LogError(ex.Message);
}
}
await ExecuteBatch(table);
return(returnTotalRows);
}
public static async Task <IActionResult> SeedDatabase(
[HttpTrigger(AuthorizationLevel.Anonymous, Global.HttpVerbGet, Route = null)] HttpRequest req,
[Table(Global.UserTableName, Connection = Global.AzureWebJobsStorage)] CloudTable userCloudTable,
ILogger log)
{
var userItems = new List <UserItem>();
TableContinuationToken continuationToken = null;
var entities = await userCloudTable.ExecuteQuerySegmentedAsync <UserEntity>(new TableQuery <UserEntity>(), continuationToken);
if (entities.Results.Count > 0)
{
foreach (var userEntity in entities.Results)
{
userItems.Add(userEntity.ToUserItem());
log.LogInformation(String.Format(GlobalMessages.UserAlreadyInDatabaseFormat, userEntity.UserName));
}
return(new OkObjectResult(userItems));
}
// database is empty - now seed it, this simulates a live system with users.
var userA = MakeUserEntity("John", "john", "56bc6a96-d6dc-406e-b36f-46373936c3bd");
var userB = MakeUserEntity("Sue", "sue", "8f9b0fa5-4afe-4120-9e99-4f32148a79fc");
var userC = MakeUserEntity("Tim", "tim", "197f8915-a37f-41d0-97da-5514d85966b5");
var batchOperation = new TableBatchOperation();
batchOperation.InsertOrReplace(userA);
batchOperation.InsertOrReplace(userB);
batchOperation.InsertOrReplace(userC);
var results = await userCloudTable.ExecuteBatchAsync(batchOperation);
foreach (var result in results)
{
if (result.Result is UserEntity userEntity)
{
userItems.Add(userEntity.ToUserItem());
log.LogInformation(String.Format(GlobalMessages.UserAddedToDatabaseFormat, userEntity.UserName));
}
}
return(new OkObjectResult(userItems));
}
protected override async Task PushToTargetAsync(IEnumerable <ResourceUsageRecord> records, Guid batchId, CancellationToken cancellationToken)
{
try
{
var batchOperation = new TableBatchOperation();
foreach (var record in records)
{
// Fill in record with Tenant info
var tenant = await TenantCacheClient.GetTenantAsync(record.EngagementAccount);
if (tenant == null)
{
BillingEventSource.Current.Warning(BillingEventSource.EmptyTrackingId, this, nameof(this.PushToTargetAsync), OperationStates.FailedMatch, $"Unable to get tenant for account '{record.EngagementAccount}'");
}
else if (this.pushUsageForWhitelistedSubscriptionsOnly)
{
if (Guid.TryParse(tenant.SubscriptionId, out Guid subId) && this.whitelistedSubscriptionIds.Contains(subId))
{
batchOperation.InsertOrReplace(new UsageRecord(record, tenant, batchId));
}
else
{
BillingEventSource.Current.Info(BillingEventSource.EmptyTrackingId, this, nameof(this.PushToTargetAsync), OperationStates.Dropped, $"Drop usage for subscription '{tenant.SubscriptionId}' as it's not in whitelist");
}
}
else
{
batchOperation.InsertOrReplace(new UsageRecord(record, tenant, batchId));
}
}
if (batchOperation.Count > 0)
{
var results = await this.table.ExecuteBatchAsync(batchOperation, cancellationToken);
BillingEventSource.Current.Info(BillingEventSource.EmptyTrackingId, this, nameof(this.PushToTargetAsync), OperationStates.Succeeded, $"Push usage with PartitionId = {batchId}, count = {records.Count()}");
}
}
catch (Exception ex)
{
BillingEventSource.Current.ErrorException(BillingEventSource.EmptyTrackingId, this, nameof(this.PushToTargetAsync), OperationStates.Failed, string.Empty, ex);
throw ex;
}
}
public async Task UpdateWaterMarkedRender(List <WaterMarkedRender> renderList)
{
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (var render in renderList)
{
batchOperation.InsertOrReplace(new UnifiedResponse.TWaterMarkedRender(render));
}
await _WaterMarkRenderTable.ExecuteBatchAsync(batchOperation);
}
public static void SaveToStorage(IEnumerable <TemplateModel> templates)
{
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (TemplateModel template in templates)
{
batchOperation.InsertOrReplace(template);
}
StorageFactory.Instance.IpcAzureAppTenantStateTable.ExecuteBatch(batchOperation, tableReqOptions);
}
private void PersistQueryTable(IEnumerable <TwitterSearchRow> queries)
{
var batchOperation = new TableBatchOperation();
foreach (var query in queries)
{
batchOperation.InsertOrReplace(query);
}
MetadataTable.ExecuteBatch(batchOperation);
}
public void BatchInsertObjectIntoTable(params TableEntity[] objs)
{
var batchOperation = new TableBatchOperation();
foreach (TableEntity ent in objs)
{
batchOperation.InsertOrReplace(ent);
}
_table.ExecuteBatch(batchOperation);
}
public static void Run([TimerTrigger("0 */1 * * * *")] TimerInfo myTimer, TraceWriter log)
{
log.Info($"C# Timer trigger function executed at: {DateTime.Now}");
List <string> contactList;
TableBatchOperation batchOperationObj;
UserEntity userEntityObj;
int UserIncrementCounter = 0;
try
{
//Getting Mock List Data
contactList = GetMockContactnumbers();
//Authenticating Table and creating Table instance
CloudTable cloudTable = AuthTable();
//Creating Batches of 100 items in order to insert them into AzureStorage
var batches = contactList.Batch(100);
log.Info($"Batch Job Started : {DateTime.Now}");
#region BATCH JOB
//Iterating through each batch
foreach (var batch in batches)
{
batchOperationObj = new TableBatchOperation();
//Iterating through each batch entities
foreach (var item in batch)
{
userEntityObj = new UserEntity("HR Department", "rowKey");
{
UserIncrementCounter++;
userEntityObj.PartitionKey = "HR Department";
userEntityObj.RowKey = "User" + UserIncrementCounter.ToString();
userEntityObj.ContactNumber = item;
}
batchOperationObj.InsertOrReplace(userEntityObj);
}
cloudTable.ExecuteBatch(batchOperationObj);
}
#endregion BATCH JOB
log.Info($"Batch Job Completed : {DateTime.Now}");
}
catch (Exception e)
{
throw e;
}
}
private void FillBookstore()
{
TableBatchOperation operations = new TableBatchOperation();
Book b1 = new Book("001");
b1.Name = "Na drini cuprija";
b1.Price = new Random().Next(500, 1000);
b1.Count = new Random().Next(10);
Book b2 = new Book("002");
b2.Name = "Besnilo";
b2.Price = new Random().Next(500, 1000);
b2.Count = new Random().Next(10);
Book b3 = new Book("003");
b3.Name = "Lovac u zitu";
b3.Price = new Random().Next(500, 1000);
b3.Count = new Random().Next(10);
Book b4 = new Book("004");
b4.Name = "Alhemicar";
b4.Price = new Random().Next(500, 1000);
b4.Count = new Random().Next(10);
Book b5 = new Book("005");
b5.Name = "Ja,Aleks Kros";
b5.Price = new Random().Next(500, 1000);
b5.Count = new Random().Next(10);
operations.InsertOrReplace(b1);
operations.InsertOrReplace(b2);
operations.InsertOrReplace(b3);
operations.InsertOrReplace(b4);
operations.InsertOrReplace(b5);
_table.ExecuteBatch(operations);
}
public void SaveLeaves(List <Leaf> leaves)
{
var batchOp = new TableBatchOperation();
foreach (Leaf leaf in leaves)
{
batchOp.InsertOrReplace(leaf);
}
var result = table.ExecuteBatch(batchOp);
}
/// <summary>
/// Create or update an entity in the table storage.
/// </summary>
/// <param name="entities">Entities to be created or updated.</param>
/// <returns>A task that represents the work queued to execute.</returns>
protected virtual async Task CreateOrUpdateBatchAsync(List <T> entities)
{
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (var entity in entities)
{
batchOperation.InsertOrReplace(entity);
}
await this.Table.ExecuteBatchAsync(batchOperation).ConfigureAwait(true);
}
public async Task StoreAsync <T>(nStoreOperation storageOperationType, IEnumerable <T> models) where T : new()
{
try
{
// Retrieve a reference to the table.
CloudTable table = GetTableReference(GetTableName <T>());
// Create the batch operation.
TableBatchOperation batchOperation = new TableBatchOperation();
// lookup the entitymapper
var entityMapper = _entityMapperRegistry[typeof(T)];
// Add all items
foreach (var model in models)
{
switch (storageOperationType)
{
case nStoreOperation.insertOperation:
batchOperation.Insert(new DynamicTableEntity <T>(model, entityMapper));
break;
case nStoreOperation.insertOrReplaceOperation:
batchOperation.InsertOrReplace(new DynamicTableEntity <T>(model, entityMapper));
break;
case nStoreOperation.mergeOperation:
batchOperation.Merge(new DynamicTableEntity <T>(model, entityMapper));
break;
case nStoreOperation.mergeOrInserOperation:
batchOperation.InsertOrMerge(new DynamicTableEntity <T>(model, entityMapper));
break;
}
}
// execute
await table.ExecuteBatchAsync(batchOperation);
}
catch (StorageException ex)
{
// check the exception
if (!_autoCreateTable || !ex.Message.StartsWith("0:The table specified does not exist", StringComparison.CurrentCulture))
{
throw ex;
}
// try to create the table
await CreateTableAsync <T>();
// retry
await StoreAsync <T>(storageOperationType, models);
}
}
/// <summary>
/// This Function will insert entity
/// </summary>
/// <returns></returns>
public async static Task <bool> InsertEntityInBatch(CloudTable cloudTable)
{
Employee employee1 = new Employee("Training", Guid.NewGuid().ToString())
{
Email = "*****@*****.**", PhoneNumber = 123
};
Employee employee2 = new Employee("Training", Guid.NewGuid().ToString())
{
Email = "*****@*****.**", PhoneNumber = 123
};
TableBatchOperation tablebatchOperations = new TableBatchOperation();
tablebatchOperations.InsertOrReplace(employee1);
tablebatchOperations.InsertOrReplace(employee2);
await cloudTable.ExecuteBatchAsync(tablebatchOperations);
return(true);
}
/// <summary>1. Get the List of scale sets for the resource group.
/// 2. Get all the virtual machines from the each scale set and them into batch operation
/// 3. Combine all the tasks and return the list of tasks.</summary>
/// <param name="virtualMachineScaleSets">List of scale sets for the resource group</param>
/// <param name="batchTasks"></param>
/// <param name="virtualMachineCloudTable">Get the virtual machine table instance</param>
/// <param name="virtualMachineScaleSetCloudTable">Get the scale set table instance</param>
/// <param name="log">Trace writer instance</param>
/// <returns></returns>
private static void GetVirtualMachineAndLoadBalancerBatch(IEnumerable <ILoadBalancer> loadBalancers,
ConcurrentBag <Task> batchTasks, CloudTable virtualMachineCloudTable,
CloudTable loadBalancerCloudTable, AzureClient azureClient, TraceWriter log)
{
if (loadBalancers == null || virtualMachineCloudTable == null || loadBalancerCloudTable == null)
{
return;
}
var listOfLoadBalancerEntities = new ConcurrentBag <LoadBalancerCrawlerResponse>();
// get the batch operation for all the load balancers and corresponding virtual machine instances
//foreach (var eachLoadBalancer in loadBalancers)
Parallel.ForEach(loadBalancers, eachLoadBalancer =>
{
try
{
listOfLoadBalancerEntities.Add(ConvertToLoadBalancerCrawlerResponse(eachLoadBalancer, azureClient, log));
var loadBalancersVirtualMachines = GetVirtualMachinesFromLoadBalancers(eachLoadBalancer.ResourceGroupName, eachLoadBalancer.Id, azureClient, log);
var tasks = new List <Task>
{
loadBalancersVirtualMachines
};
Task.WhenAll(tasks);
var virtualMachineIds = loadBalancersVirtualMachines.Result;
var virtualMachinesBatchOperation = InsertLoadBalancerVirtualMachines(virtualMachineIds, eachLoadBalancer, azureClient, log);
if (virtualMachinesBatchOperation != null && virtualMachinesBatchOperation.Count > 0)
{
batchTasks.Add(virtualMachineCloudTable.ExecuteBatchAsync(virtualMachinesBatchOperation));
}
var virtualMachinesList = new List <string>();
}
catch (Exception e)
{
// catch the error, to continue adding other entities to table
log.Error($"timercrawlerforvirtualmachinescaleset threw the exception ", e,
"GetVirtualMachineAndScaleSetBatch for the scale set: " + eachLoadBalancer.Name);
}
});
// table batch operation currently allows only 100 per batch, So ensuring the one batch operation will have only 100 items
for (var i = 0; i < listOfLoadBalancerEntities.Count; i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = listOfLoadBalancerEntities.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
var loadBalancerTableBatchOperation = new TableBatchOperation();
foreach (var entity in batchItems)
{
loadBalancerTableBatchOperation.InsertOrReplace(entity);
}
batchTasks.Add(loadBalancerCloudTable.ExecuteBatchAsync(loadBalancerTableBatchOperation));
}
}
public void TableBatchAllSupportedOperationsSync()
{
TableBatchOperation batch = new TableBatchOperation();
string pk = Guid.NewGuid().ToString();
// insert
batch.Insert(GenerateRandomEnitity(pk));
// delete
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.Delete(entity);
}
// replace
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.Replace(entity);
}
// insert or replace
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.InsertOrReplace(entity);
}
// merge
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.Merge(entity);
}
// insert or merge
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.InsertOrMerge(entity);
}
IList <TableResult> results = this.repTable.ExecuteBatch(batch);
Assert.AreEqual(results.Count, 6);
IEnumerator <TableResult> enumerator = results.GetEnumerator();
for (int i = 0; i < results.Count; i++)
{
enumerator.MoveNext();
Assert.AreEqual((int)HttpStatusCode.NoContent, enumerator.Current.HttpStatusCode, "HttpStatusCode mismatch i={0}", i);
}
}
public async Task AddOrUpdateAsync(IEnumerable <T> objs)
{
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (var obj in objs)
{
batchOperation.InsertOrReplace(obj);
}
await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false);
}
public static Task InsertOrMergeAll(this CloudTable table, IEnumerable <TableEntity> entities)
{
var batchOp = new TableBatchOperation();
foreach (var entity in entities)
{
batchOp.InsertOrReplace(entity);
}
return(table.ExecuteBatchAsync(batchOp));
}
// All must have the same partition key
public async Task WriteBatchAsync(T[] entities, TableInsertMode mode = TableInsertMode.Insert)
{
if (entities.Length == 0)
{
return; // nothing to write.
}
string partitionKey = entities[0].PartitionKey;
const int BatchSize = 99;
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (var entity in entities)
{
if (entity.PartitionKey != partitionKey)
{
throw new InvalidOperationException("All entities in a batch must have same partition key");
}
ValidateRowKey(entity.RowKey);
switch (mode)
{
case TableInsertMode.Insert:
batchOperation.Insert(entity);
break;
case TableInsertMode.InsertOrMerge:
batchOperation.InsertOrMerge(entity);
break;
case TableInsertMode.InsertOrReplace:
batchOperation.InsertOrReplace(entity);
break;
default:
throw new InvalidOperationException("Unsupported insert mode: " + mode.ToString());
}
if (batchOperation.Count == BatchSize)
{
// Flush
await _table.ExecuteBatchAsync(batchOperation);
batchOperation = new TableBatchOperation();
}
}
if (batchOperation.Count > 0)
{
await _table.ExecuteBatchAsync(batchOperation);
}
}
private TableBatchOperation BuildBatch <T>(IEnumerable <T> records, string partition, bool isDelete = false)
{
TableBatchOperation batch = new TableBatchOperation();
foreach (var record in records)
{
if (record is ITableEntity)
{
if (string.IsNullOrEmpty((record as ITableEntity).PartitionKey))
{
(record as ITableEntity).PartitionKey = partition;
if (isDelete)
{
(record as ITableEntity).ETag = "*";
}
}
if (!isDelete)
{
batch.InsertOrReplace((ITableEntity)record);
}
if (isDelete)
{
batch.Delete((ITableEntity)record);
}
}
else
{
var obj = BuildTableEntity(record);
if (!isDelete)
{
batch.InsertOrReplace((ITableEntity)obj);
}
if (isDelete)
{
batch.Delete((ITableEntity)obj);
}
}
}
return(batch);
}
private IList <AcceptedUserComment> addComments(string type, int dddEventId, DDDEventDetail eventDetail, string userName, string userToken, string clientToken, IList <UserComment> userComments)
{
if (userComments == null || userComments.Count == 0)
{
return(null);
}
var eventScoreService = LazyLoadEventScoreService(userToken, userName, dddEventId);
var token = userToken ?? clientToken;
var comments = userComments.Select(x => new Comment
{
PartitionKey = dddEventId.ToString(),
RowKey = type + token + "_" + x.id + "",
SessionId = x.sessionId,
Type = type,
UserName = userName,
UserToken = userToken,
ClientToken = clientToken,
UserComment = x.comment,
Date = x.date
});
foreach (var comment in userComments)
{
if (comment.sessionId != null)
{
var session = eventDetail.Sessions.First(x => x.Id == comment.sessionId);
eventScoreService.AddCommentItem(type, session.TimeSlotId);
}
else
{
eventScoreService.AddCommentItem(type, null);
}
}
TableBatchOperation batch = new TableBatchOperation();
foreach (var comment in comments)
{
batch.InsertOrReplace(comment);
}
userCommentsTable.ExecuteBatch(batch);
var accepted = userComments.Select(x => new AcceptedUserComment
{
eventId = dddEventId,
id = x.id,
sessionId = x.sessionId
});
return(accepted.ToList());
}
private void addOrUpdateSessionData(int dddEventId, DDDEventDetail eventDetail, string userName, string userToken, string clientToken, IList <PocketDDD.Models.UserSessionData> sessionData)
{
if (sessionData == null || sessionData.Count() == 0)
{
return;
}
var eventScoreService = LazyLoadEventScoreService(userToken, userName, dddEventId);
var token = userToken ?? clientToken;
var userSessionDatas = sessionData.Select(x => new PocketDDD.Models.Azure.UserSessionData
{
PartitionKey = dddEventId.ToString(),
RowKey = x.sessionId.ToString() + "_" + token,
SessionId = x.sessionId,
UserName = userName,
UserToken = userToken,
ClientToken = clientToken,
Bookmarked = x.bookmarked,
AttendingStatus = x.attendingStatus,
SpeakerKnowledgeRating = x.speakerKnowledgeRating,
SpeakerSkillsRating = x.speakerSkillsRating
});
foreach (var sessionDataItem in sessionData)
{
var session = eventDetail.Sessions.First(x => x.Id == sessionDataItem.sessionId);
if (sessionDataItem.bookmarked || sessionDataItem.attendingStatus != 2)
{
eventScoreService.AddBookmarkOrAttendingItem(session.TimeSlotId);
}
if (sessionDataItem.speakerKnowledgeRating != null)
{
eventScoreService.AddKnowledgeRatingItem(session.TimeSlotId);
}
if (sessionDataItem.speakerSkillsRating != null)
{
eventScoreService.AddSkillRatingItem(session.TimeSlotId);
}
}
TableBatchOperation batch = new TableBatchOperation();
foreach (var userSessionData in userSessionDatas)
{
batch.InsertOrReplace(userSessionData);
}
userSessionDataTable.ExecuteBatch(batch);
}
public void InsertOrReplaceBatch(IEnumerable <T> objs)
{
List <List <T> > chunks = GetChunks(objs);
foreach (var chunk in chunks)
{
var batchOperation = new TableBatchOperation();
foreach (var obj in chunk)
{
batchOperation.InsertOrReplace(obj);
}
Table.ExecuteBatch(batchOperation);
}
}
public void UpdatePlayers(IEnumerable <PlayerEntity> players)
{
var tableClient = GetClient();
CloudTable table = tableClient.GetTableReference("players");
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (var player in players)
{
batchOperation.InsertOrReplace(player);
}
table.ExecuteBatch(batchOperation);
}
static TableBatchOperation SaveBootstrap(IEnumerable<SearchModel> searchModel)
{
var batchOperation = new TableBatchOperation();
foreach (var model in searchModel)
{
if (string.IsNullOrWhiteSpace(model.Url) || string.IsNullOrWhiteSpace(model.PartitionKey))
{
throw new InvalidOperationException("We require a partition key, and row key");
}
batchOperation.InsertOrReplace(model);
}
return batchOperation;
}
public IList <TableResult> UpsertEntities(string tableName, IList <StorageEntity> entities)
{
SetProperties(entities);
CloudTable table = _storage.GetTableReference(tableName);
table.CreateIfNotExists();
TableBatchOperation operation = new TableBatchOperation();
foreach (StorageEntity entity in entities)
{
operation.InsertOrReplace(entity);
}
return(table.ExecuteBatch(operation));
}
public void Load()
{
TableBatchManager.LogInfo("Load batch called");
try
{
TableBatchOperation op = new TableBatchOperation();
foreach (var e in Entities)
{
if (Merge)
op.InsertOrMerge(e);
else
op.InsertOrReplace(e);
}
Table.ExecuteBatch(op);
TableBatchManager.LogInfo("exec batch");
}
catch(Exception ex)
{
TableBatchManager.LogError("Error: " + ex);
}
}
public static void ProcessPlayerGame(Int64 playerId, PlayerGameData gameData, Guid gameId, Int64 gameSeconds)
{
// Create the batch operation.
TableBatchOperation batchOperation = new TableBatchOperation();
//TODO: Get Player Row
PlayerEntity player = PlayerEntity.GetPlayerEntity(playerId);
batchOperation.InsertOrReplace(player);
//Update Player Entity with Game Data
player.TotalDeaths += gameData.Deaths;
player.TotalKills += gameData.Kills;
player.TotalPoints += gameData.Points;
player.TotalWins += gameData.Win ? 1 : 0;
player.TotalGames += 1;
player.TotalSecondsPlayed += gameSeconds;
//Create PlayerGame Row
PlayerGameEntity playerGame = new PlayerGameEntity(playerId, gameId)
{
Points = gameData.Points,
Win = gameData.Win,
Kills = gameData.Kills,
Deaths = gameData.Deaths,
GameDuration = gameSeconds
};
batchOperation.Insert(playerGame);
try
{
StorageManager.Instance.PlayersTable.ExecuteBatch(batchOperation);
}
catch (Exception ex)
{
//TODO: handle exception, check if its because an entity already existed.
//This means we've already handled this data.
}
}
public async Task<bool> AddSubscriptions(IEnumerable<UserSubscription> subscriptionsToAdd)
{
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable userSubscriptionsTable =
tableClient.GetTableReference("userSubscriptions");
var tableExists = await userSubscriptionsTable.ExistsAsync();
if (!tableExists)
{
await userSubscriptionsTable.CreateIfNotExistsAsync();
}
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (var subscription in subscriptionsToAdd)
{
UserSubscriptionEntity userSubscriptionEntity =
new UserSubscriptionEntity(subscription.UserId, subscription.FriendId);
batchOperation.InsertOrReplace(userSubscriptionEntity);
}
await userSubscriptionsTable.ExecuteBatchAsync(batchOperation);
return true;
}
public int CommitChanges()
{
if (string.IsNullOrEmpty(this.token))
return 0;
if(ScoreItems.Count > 0)
{
TableBatchOperation batch = new TableBatchOperation();
ScoreItems.ToList().ForEach(x =>
{
batch.InsertOrReplace(x.Value);
});
eventScoreTable.ExecuteBatch(batch);
}
TableQuery<EventScoreItem> query = new TableQuery<EventScoreItem>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, this.token));
var eventScore = eventScoreTable.ExecuteQuery(query).Sum(x=>x.Score);
var totalItem = new EventScoreItem
{
PartitionKey = dddEventId.ToString(),
RowKey = this.token,
DDDEventId = dddEventId,
UserName = userName,
Score = eventScore
};
TableOperation insert = TableOperation.InsertOrReplace(totalItem);
eventScoreTable.Execute(insert);
return eventScore;
}
private IList<AcceptedUserComment> addComments(string type, int dddEventId, DDDEventDetail eventDetail, string userName, string userToken, string clientToken, IList<UserComment> userComments)
{
if (userComments == null || userComments.Count == 0)
return null;
var eventScoreService = LazyLoadEventScoreService(userToken, userName, dddEventId);
var token = userToken ?? clientToken;
var comments = userComments.Select(x => new Comment
{
PartitionKey = dddEventId.ToString(),
RowKey = type + token + "_" + x.id + "",
SessionId = x.sessionId,
Type = type,
UserName = userName,
UserToken = userToken,
ClientToken = clientToken,
UserComment = x.comment,
Date = x.date
});
foreach (var comment in userComments)
{
if (comment.sessionId != null)
{
var session = eventDetail.Sessions.First(x => x.Id == comment.sessionId);
eventScoreService.AddCommentItem(type, session.TimeSlotId);
}
else
{
eventScoreService.AddCommentItem(type, null);
}
}
TableBatchOperation batch = new TableBatchOperation();
foreach (var comment in comments)
{
batch.InsertOrReplace(comment);
}
userCommentsTable.ExecuteBatch(batch);
var accepted = userComments.Select(x => new AcceptedUserComment
{
eventId = dddEventId,
id = x.id,
sessionId = x.sessionId
});
return accepted.ToList();
}
public void TableBatchAllSupportedOperationsSync()
{
TableBatchOperation batch = new TableBatchOperation();
string pk = Guid.NewGuid().ToString();
// insert
batch.Insert(GenerateRandomEnitity(pk));
// delete
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.Delete(entity);
}
// replace
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.Replace(entity);
}
// insert or replace
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.InsertOrReplace(entity);
}
// merge
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.Merge(entity);
}
// insert or merge
{
DynamicReplicatedTableEntity entity = GenerateRandomEnitity(pk);
this.repTable.Execute(TableOperation.Insert(entity));
batch.InsertOrMerge(entity);
}
IList<TableResult> results = this.repTable.ExecuteBatch(batch);
Assert.AreEqual(results.Count, 6);
IEnumerator<TableResult> enumerator = results.GetEnumerator();
for (int i = 0; i < results.Count; i++)
{
enumerator.MoveNext();
Assert.AreEqual((int)HttpStatusCode.NoContent, enumerator.Current.HttpStatusCode, "HttpStatusCode mismatch i={0}", i);
}
}
public async Task Append(string eventStreamId, string transactionId, int expectedVersion, IEnumerable<IAggregateEvent<JObject>> events)
{
var table = this.client.GetTableReference(this.tableName);
await EnsureTableExists(table);
// no need to validate expected version, because the batch transaction will fail if a duplicate exists
var transaction = new TableBatchOperation();
transaction.InsertOrReplace(CreateSourceRoot(eventStreamId, expectedVersion + events.Count()));
foreach (var ev in events)
{
transaction.Insert(ev.ToEntity());
}
transaction.Insert(CreateUncommittedLog(eventStreamId, transactionId, events.Select(ev => ev.SequenceId)));
try
{
var results = await table.ExecuteBatchAsync(transaction);
return;
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode != (int)HttpStatusCode.Conflict)
{
throw; // different error, rethrow
}
}
// conflict
var sourceRootVersion = await GetSourceRootVersion(table, eventStreamId);
throw new ConcurrencyException(eventStreamId, expectedVersion, sourceRootVersion);
}
private async Task SaveInfo()
{
CloudTable metatable = _dstClient.GetTableReference(MetatableName);
Console.WriteLine("new destination status");
var op = new TableBatchOperation();
foreach (var batch in _timestamps.OrderBy(x => x.Key.Name).Batch(100))
{
foreach (var item in batch)
{
var entity = new DynamicTableEntity("SyncTimestamps", item.Key.Name);
entity.Properties["SourceTimestamp"] = new EntityProperty(item.Value);
op.InsertOrReplace(entity);
Console.WriteLine("{0,-40}{1}", item.Key.Name, item.Value);
}
await metatable.ExecuteBatchAsync(op, _token);
}
}
private async Task DoTableBatchOperationsWithEmptyKeysAsync(TablePayloadFormat format)
{
tableClient.DefaultRequestOptions.PayloadFormat = format;
// Insert Entity
DynamicTableEntity ent = new DynamicTableEntity() { PartitionKey = "", RowKey = "" };
ent.Properties.Add("foo2", new EntityProperty("bar2"));
ent.Properties.Add("foo", new EntityProperty("bar"));
TableBatchOperation batch = new TableBatchOperation();
batch.Insert(ent);
await currentTable.ExecuteBatchAsync(batch);
// Retrieve Entity
TableBatchOperation retrieveBatch = new TableBatchOperation();
retrieveBatch.Retrieve(ent.PartitionKey, ent.RowKey);
TableResult result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First();
DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey);
Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey);
Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(ent.Properties["foo"].StringValue, retrievedEntity.Properties["foo"].StringValue);
Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]);
Assert.AreEqual(ent.Properties["foo2"].StringValue, retrievedEntity.Properties["foo2"].StringValue);
Assert.AreEqual(ent.Properties["foo2"], retrievedEntity.Properties["foo2"]);
// InsertOrMerge
DynamicTableEntity insertOrMergeEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey);
insertOrMergeEntity.Properties.Add("foo3", new EntityProperty("value"));
batch = new TableBatchOperation();
batch.InsertOrMerge(insertOrMergeEntity);
await currentTable.ExecuteBatchAsync(batch);
result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(insertOrMergeEntity.Properties["foo3"], retrievedEntity.Properties["foo3"]);
// InsertOrReplace
DynamicTableEntity insertOrReplaceEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey);
insertOrReplaceEntity.Properties.Add("prop2", new EntityProperty("otherValue"));
batch = new TableBatchOperation();
batch.InsertOrReplace(insertOrReplaceEntity);
await currentTable.ExecuteBatchAsync(batch);
result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(1, retrievedEntity.Properties.Count);
Assert.AreEqual(insertOrReplaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]);
// Merge
DynamicTableEntity mergeEntity = new DynamicTableEntity(retrievedEntity.PartitionKey, retrievedEntity.RowKey) { ETag = retrievedEntity.ETag };
mergeEntity.Properties.Add("mergeProp", new EntityProperty("merged"));
batch = new TableBatchOperation();
batch.Merge(mergeEntity);
await currentTable.ExecuteBatchAsync(batch);
// Retrieve Entity & Verify Contents
result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(mergeEntity.Properties["mergeProp"], retrievedEntity.Properties["mergeProp"]);
// Replace
DynamicTableEntity replaceEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey) { ETag = retrievedEntity.ETag };
replaceEntity.Properties.Add("replaceProp", new EntityProperty("replace"));
batch = new TableBatchOperation();
batch.Replace(replaceEntity);
await currentTable.ExecuteBatchAsync(batch);
// Retrieve Entity & Verify Contents
result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["replaceProp"], retrievedEntity.Properties["replaceProp"]);
// Delete Entity
batch = new TableBatchOperation();
batch.Delete(retrievedEntity);
await currentTable.ExecuteBatchAsync(batch);
// Retrieve Entity
result = (await currentTable.ExecuteBatchAsync(retrieveBatch)).First();
Assert.IsNull(result.Result);
}
public void TableBatchInsertOrReplaceAPM()
{
CloudTableClient tableClient = GenerateCloudTableClient();
// Insert Or Replace with no pre-existing entity
DynamicTableEntity insertOrReplaceEntity = new DynamicTableEntity("insertOrReplace entity", "foo");
insertOrReplaceEntity.Properties.Add("prop1", new EntityProperty("value1"));
TableBatchOperation batch = new TableBatchOperation();
batch.InsertOrReplace(insertOrReplaceEntity);
using (ManualResetEvent evt = new ManualResetEvent(false))
{
IAsyncResult asyncRes = null;
currentTable.BeginExecuteBatch(batch, (res) =>
{
asyncRes = res;
evt.Set();
}, null);
evt.WaitOne();
currentTable.EndExecuteBatch(asyncRes);
}
// Retrieve Entity & Verify Contents
TableResult result = currentTable.Execute(TableOperation.Retrieve(insertOrReplaceEntity.PartitionKey, insertOrReplaceEntity.RowKey));
DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(insertOrReplaceEntity.Properties.Count, retrievedEntity.Properties.Count);
DynamicTableEntity replaceEntity = new DynamicTableEntity(insertOrReplaceEntity.PartitionKey, insertOrReplaceEntity.RowKey);
replaceEntity.Properties.Add("prop2", new EntityProperty("value2"));
TableBatchOperation batch2 = new TableBatchOperation();
batch2.InsertOrReplace(replaceEntity);
using (ManualResetEvent evt = new ManualResetEvent(false))
{
IAsyncResult asyncRes = null;
currentTable.BeginExecuteBatch(batch2, (res) =>
{
asyncRes = res;
evt.Set();
}, null);
evt.WaitOne();
currentTable.EndExecuteBatch(asyncRes);
}
// Retrieve Entity & Verify Contents
result = currentTable.Execute(TableOperation.Retrieve(insertOrReplaceEntity.PartitionKey, insertOrReplaceEntity.RowKey));
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(1, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]);
}
public void TableBatchOperationsWithEmptyKeys()
{
CloudTableClient tableClient = GenerateCloudTableClient();
// Insert Entity
DynamicTableEntity ent = new DynamicTableEntity() { PartitionKey = "", RowKey = "" };
ent.Properties.Add("foo2", new EntityProperty("bar2"));
ent.Properties.Add("foo", new EntityProperty("bar"));
TableBatchOperation batch = new TableBatchOperation();
batch.Insert(ent);
currentTable.ExecuteBatch(batch);
// Retrieve Entity
TableBatchOperation retrieveBatch = new TableBatchOperation();
retrieveBatch.Retrieve(ent.PartitionKey, ent.RowKey);
TableResult result = currentTable.ExecuteBatch(retrieveBatch).First();
DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(ent.PartitionKey, retrievedEntity.PartitionKey);
Assert.AreEqual(ent.RowKey, retrievedEntity.RowKey);
Assert.AreEqual(ent.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(ent.Properties["foo"].StringValue, retrievedEntity.Properties["foo"].StringValue);
Assert.AreEqual(ent.Properties["foo"], retrievedEntity.Properties["foo"]);
Assert.AreEqual(ent.Properties["foo2"].StringValue, retrievedEntity.Properties["foo2"].StringValue);
Assert.AreEqual(ent.Properties["foo2"], retrievedEntity.Properties["foo2"]);
// InsertOrMerge
DynamicTableEntity insertOrMergeEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey);
insertOrMergeEntity.Properties.Add("foo3", new EntityProperty("value"));
batch = new TableBatchOperation();
batch.InsertOrMerge(insertOrMergeEntity);
currentTable.ExecuteBatch(batch);
result = currentTable.ExecuteBatch(retrieveBatch).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(insertOrMergeEntity.Properties["foo3"], retrievedEntity.Properties["foo3"]);
// InsertOrReplace
DynamicTableEntity insertOrReplaceEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey);
insertOrReplaceEntity.Properties.Add("prop2", new EntityProperty("otherValue"));
batch = new TableBatchOperation();
batch.InsertOrReplace(insertOrReplaceEntity);
currentTable.ExecuteBatch(batch);
result = currentTable.ExecuteBatch(retrieveBatch).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(1, retrievedEntity.Properties.Count);
Assert.AreEqual(insertOrReplaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]);
// Merge
DynamicTableEntity mergeEntity = new DynamicTableEntity(retrievedEntity.PartitionKey, retrievedEntity.RowKey) { ETag = retrievedEntity.ETag };
mergeEntity.Properties.Add("mergeProp", new EntityProperty("merged"));
batch = new TableBatchOperation();
batch.Merge(mergeEntity);
currentTable.ExecuteBatch(batch);
// Retrieve Entity & Verify Contents
result = currentTable.ExecuteBatch(retrieveBatch).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(mergeEntity.Properties["mergeProp"], retrievedEntity.Properties["mergeProp"]);
// Replace
DynamicTableEntity replaceEntity = new DynamicTableEntity(ent.PartitionKey, ent.RowKey) { ETag = retrievedEntity.ETag };
replaceEntity.Properties.Add("replaceProp", new EntityProperty("replace"));
batch = new TableBatchOperation();
batch.Replace(replaceEntity);
currentTable.ExecuteBatch(batch);
// Retrieve Entity & Verify Contents
result = currentTable.ExecuteBatch(retrieveBatch).First();
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(replaceEntity.Properties.Count, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["replaceProp"], retrievedEntity.Properties["replaceProp"]);
// Delete Entity
batch = new TableBatchOperation();
batch.Delete(retrievedEntity);
currentTable.ExecuteBatch(batch);
// Retrieve Entity
result = currentTable.ExecuteBatch(retrieveBatch).First();
Assert.IsNull(result.Result);
}
public void TableBatchAllSupportedOperationsSync()
{
CloudTableClient tableClient = GenerateCloudTableClient();
TableBatchOperation batch = new TableBatchOperation();
string pk = Guid.NewGuid().ToString();
// insert
batch.Insert(GenerateRandomEnitity(pk));
// delete
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.Delete(entity);
}
// replace
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.Replace(entity);
}
// insert or replace
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.InsertOrReplace(entity);
}
// merge
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.Merge(entity);
}
// insert or merge
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.InsertOrMerge(entity);
}
IList<TableResult> results = currentTable.ExecuteBatch(batch);
Assert.AreEqual(results.Count, 6);
IEnumerator<TableResult> enumerator = results.GetEnumerator();
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.Created);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
}
async Task DoRandomAtomicCalls()
{
for (int callNum = 0; callNum < MigrationModel.NUM_CALLS_PER_MACHINE; callNum++)
{
SortedDictionary<PrimaryKey, DynamicTableEntity> dump = await peekProxy.DumpReferenceTableAsync();
if (PSharpRuntime.Nondeterministic())
{
// Query
var query = new TableQuery<DynamicTableEntity>();
query.FilterString = ChainTableUtils.CombineFilters(
TableQuery.GenerateFilterCondition(
TableConstants.PartitionKey, QueryComparisons.Equal, MigrationModel.SINGLE_PARTITION_KEY),
TableOperators.And,
NondeterministicUserPropertyFilterString());
await RunQueryAtomicAsync(query);
}
else
{
// Batch write
int batchSize = PSharpRuntime.Nondeterministic() ? 2 : 1;
var batch = new TableBatchOperation();
var rowKeyChoices = new List<string> { "0", "1", "2", "3", "4", "5" };
for (int opNum = 0; opNum < batchSize; opNum++)
{
int opTypeNum = PSharpNondeterminism.Choice(7);
int rowKeyI = PSharpNondeterminism.Choice(rowKeyChoices.Count);
string rowKey = rowKeyChoices[rowKeyI];
rowKeyChoices.RemoveAt(rowKeyI); // Avoid duplicate in same batch
var primaryKey = new PrimaryKey(MigrationModel.SINGLE_PARTITION_KEY, rowKey);
string eTag = null;
if (opTypeNum >= 1 && opTypeNum <= 3)
{
DynamicTableEntity existingEntity;
int etagTypeNum = PSharpNondeterminism.Choice(
dump.TryGetValue(primaryKey, out existingEntity) ? 3 : 2);
switch (etagTypeNum)
{
case 0: eTag = ChainTable2Constants.ETAG_ANY; break;
case 1: eTag = "wrong"; break;
case 2: eTag = existingEntity.ETag; break;
}
}
DynamicTableEntity entity = new DynamicTableEntity
{
PartitionKey = MigrationModel.SINGLE_PARTITION_KEY,
RowKey = rowKey,
ETag = eTag,
Properties = new Dictionary<string, EntityProperty> {
// Give us something to see on merge. Might help with tracing too!
{ string.Format("{0}_c{1}_o{2}", machineId.ToString(), callNum, opNum),
new EntityProperty(true) },
// Property with 50%/50% distribution for use in filters.
{ "isHappy", new EntityProperty(PSharpRuntime.Nondeterministic()) }
}
};
switch (opTypeNum)
{
case 0: batch.Insert(entity); break;
case 1: batch.Replace(entity); break;
case 2: batch.Merge(entity); break;
case 3: batch.Delete(entity); break;
case 4: batch.InsertOrReplace(entity); break;
case 5: batch.InsertOrMerge(entity); break;
case 6:
entity.ETag = ChainTable2Constants.ETAG_DELETE_IF_EXISTS;
batch.Delete(entity); break;
}
}
await RunBatchAsync(batch);
}
}
}
async Task LoadDefaultDataAsync()
{
#if false
MTableEntity eMeta = new MTableEntity {
PartitionKey = MigrationModel.SINGLE_PARTITION_KEY,
RowKey = MigratingTable.ROW_KEY_PARTITION_META,
partitionState = MTablePartitionState.SWITCHED,
};
MTableEntity e0 = TestUtils.CreateTestMTableEntity("0", "orange");
MTableEntity e1old = TestUtils.CreateTestMTableEntity("1", "red");
MTableEntity e2new = TestUtils.CreateTestMTableEntity("2", "green");
MTableEntity e3old = TestUtils.CreateTestMTableEntity("3", "blue");
MTableEntity e3new = TestUtils.CreateTestMTableEntity("3", "azure");
MTableEntity e4old = TestUtils.CreateTestMTableEntity("4", "yellow");
MTableEntity e4new = TestUtils.CreateTestMTableEntity("4", null, true);
var oldBatch = new TableBatchOperation();
oldBatch.InsertOrReplace(eMeta);
oldBatch.InsertOrReplace(e0);
oldBatch.InsertOrReplace(e1old);
oldBatch.InsertOrReplace(e3old);
oldBatch.InsertOrReplace(e4old);
IList<TableResult> oldTableResult = await oldTable.ExecuteBatchAsync(oldBatch);
await ExecuteExportedMirrorBatchAsync(oldBatch, oldTableResult);
var newBatch = new TableBatchOperation();
newBatch.InsertOrReplace(e0);
newBatch.InsertOrReplace(e2new);
newBatch.InsertOrReplace(e3new);
newBatch.InsertOrReplace(e4new);
IList<TableResult> newTableResult = await newTable.ExecuteBatchAsync(newBatch);
// Allow rows to overwrite rather than composing the virtual ETags manually.
// InsertOrReplace doesn't use the ETag, so we don't care that the ETag was mutated by the original batch.
await ExecuteExportedMirrorBatchAsync(newBatch, newTableResult);
#endif
// Start with the old table now.
var batch = new TableBatchOperation();
batch.InsertOrReplace(TestUtils.CreateTestEntity2("0", true));
batch.InsertOrReplace(TestUtils.CreateTestEntity2("1", false));
batch.InsertOrReplace(TestUtils.CreateTestEntity2("3", false));
batch.InsertOrReplace(TestUtils.CreateTestEntity2("4", true));
IList<TableResult> oldTableResult = await oldTable.ExecuteBatchAsync(batch);
// InsertOrReplace doesn't use the ETag, so we don't care that the ETag was mutated by the original batch.
await referenceTable.ExecuteMirrorBatchAsync(batch, oldTableResult);
}
public void TableBatchAllSupportedOperationsAPM()
{
CloudTableClient tableClient = GenerateCloudTableClient();
TableBatchOperation batch = new TableBatchOperation();
string pk = Guid.NewGuid().ToString();
// insert
batch.Insert(GenerateRandomEnitity(pk));
// delete
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.Delete(entity);
}
// replace
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.Replace(entity);
}
// insert or replace
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.InsertOrReplace(entity);
}
// merge
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.Merge(entity);
}
// insert or merge
{
DynamicTableEntity entity = GenerateRandomEnitity(pk);
currentTable.Execute(TableOperation.Insert(entity));
batch.InsertOrMerge(entity);
}
IList<TableResult> results = null;
using (ManualResetEvent evt = new ManualResetEvent(false))
{
IAsyncResult asyncRes = null;
currentTable.BeginExecuteBatch(batch, (res) =>
{
asyncRes = res;
evt.Set();
}, null);
evt.WaitOne();
results = currentTable.EndExecuteBatch(asyncRes);
}
Assert.AreEqual(results.Count, 6);
IEnumerator<TableResult> enumerator = results.GetEnumerator();
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.Created);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
}
private void addOrUpdateSessionData(int dddEventId, DDDEventDetail eventDetail, string userName, string userToken, string clientToken, IList<PocketDDD.Models.UserSessionData> sessionData)
{
if (sessionData == null || sessionData.Count() == 0)
return;
var eventScoreService = LazyLoadEventScoreService(userToken, userName, dddEventId);
var token = userToken ?? clientToken;
var userSessionDatas = sessionData.Select(x => new PocketDDD.Models.Azure.UserSessionData
{
PartitionKey = dddEventId.ToString(),
RowKey = x.sessionId.ToString() + "_" + token,
SessionId = x.sessionId,
UserName = userName,
UserToken = userToken,
ClientToken = clientToken,
Bookmarked = x.bookmarked,
AttendingStatus = x.attendingStatus,
SpeakerKnowledgeRating = x.speakerKnowledgeRating,
SpeakerSkillsRating = x.speakerSkillsRating
});
foreach (var sessionDataItem in sessionData)
{
var session = eventDetail.Sessions.First(x => x.Id == sessionDataItem.sessionId);
if (sessionDataItem.bookmarked || sessionDataItem.attendingStatus != 2)
eventScoreService.AddBookmarkOrAttendingItem(session.TimeSlotId);
if(sessionDataItem.speakerKnowledgeRating != null)
eventScoreService.AddKnowledgeRatingItem(session.TimeSlotId);
if (sessionDataItem.speakerSkillsRating != null)
eventScoreService.AddSkillRatingItem(session.TimeSlotId);
}
TableBatchOperation batch = new TableBatchOperation();
foreach (var userSessionData in userSessionDatas)
{
batch.InsertOrReplace(userSessionData);
}
userSessionDataTable.ExecuteBatch(batch);
}
public void TableBatchInsertOrReplaceSync()
{
CloudTableClient tableClient = GenerateCloudTableClient();
// Insert Or Replace with no pre-existing entity
DynamicTableEntity insertOrReplaceEntity = new DynamicTableEntity("insertOrReplace entity", "foo");
insertOrReplaceEntity.Properties.Add("prop1", new EntityProperty("value1"));
TableBatchOperation batch = new TableBatchOperation();
batch.InsertOrReplace(insertOrReplaceEntity);
currentTable.ExecuteBatch(batch);
// Retrieve Entity & Verify Contents
TableResult result = currentTable.Execute(TableOperation.Retrieve(insertOrReplaceEntity.PartitionKey, insertOrReplaceEntity.RowKey));
DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(insertOrReplaceEntity.Properties.Count, retrievedEntity.Properties.Count);
DynamicTableEntity replaceEntity = new DynamicTableEntity(insertOrReplaceEntity.PartitionKey, insertOrReplaceEntity.RowKey);
replaceEntity.Properties.Add("prop2", new EntityProperty("value2"));
TableBatchOperation batch2 = new TableBatchOperation();
batch2.InsertOrReplace(replaceEntity);
currentTable.ExecuteBatch(batch2);
// Retrieve Entity & Verify Contents
result = currentTable.Execute(TableOperation.Retrieve(insertOrReplaceEntity.PartitionKey, insertOrReplaceEntity.RowKey));
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(1, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]);
}
private void DoTableBatchInsertOrReplaceEncryption(TablePayloadFormat format)
{
tableClient.DefaultRequestOptions.PayloadFormat = format;
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
TableRequestOptions options = new TableRequestOptions()
{
EncryptionPolicy = new TableEncryptionPolicy(aesKey, null),
EncryptionResolver = (pk, rk, propName) =>
{
if (propName == "A" || propName == "B")
{
return true;
}
return false;
}
};
// Insert Or Replace with no pre-existing entity
DynamicTableEntity insertOrReplaceEntity = new DynamicTableEntity("insertOrReplace entity", "foo" + format.ToString());
insertOrReplaceEntity.Properties.Add("A", new EntityProperty("a"));
TableBatchOperation batch = new TableBatchOperation();
batch.InsertOrReplace(insertOrReplaceEntity);
currentTable.ExecuteBatch(batch, options);
// Retrieve Entity & Verify Contents
// Create the resolver to be used for unwrapping.
DictionaryKeyResolver resolver = new DictionaryKeyResolver();
resolver.Add(aesKey);
TableRequestOptions retrieveOptions = new TableRequestOptions() { EncryptionPolicy = new TableEncryptionPolicy(null, resolver) };
TableResult result = currentTable.Execute(TableOperation.Retrieve(insertOrReplaceEntity.PartitionKey, insertOrReplaceEntity.RowKey), retrieveOptions);
DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(insertOrReplaceEntity.Properties.Count, retrievedEntity.Properties.Count);
DynamicTableEntity replaceEntity = new DynamicTableEntity(insertOrReplaceEntity.PartitionKey, insertOrReplaceEntity.RowKey);
replaceEntity.Properties.Add("B", new EntityProperty("b"));
TableBatchOperation batch2 = new TableBatchOperation();
batch2.InsertOrReplace(replaceEntity);
currentTable.ExecuteBatch(batch2, options);
// Retrieve Entity & Verify Contents
result = currentTable.Execute(TableOperation.Retrieve(insertOrReplaceEntity.PartitionKey, insertOrReplaceEntity.RowKey), retrieveOptions);
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(1, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["B"], retrievedEntity.Properties["B"]);
}
private async Task DoTableBatchInsertOrReplaceAsync(TablePayloadFormat format)
{
tableClient.DefaultRequestOptions.PayloadFormat = format;
// Insert Or Replace with no pre-existing entity
DynamicTableEntity insertOrReplaceEntity = new DynamicTableEntity("insertOrReplace entity", "foo" + format.ToString());
insertOrReplaceEntity.Properties.Add("prop1", new EntityProperty("value1"));
TableBatchOperation batch = new TableBatchOperation();
batch.InsertOrReplace(insertOrReplaceEntity);
await currentTable.ExecuteBatchAsync(batch);
// Retrieve Entity & Verify Contents
TableResult result = await currentTable.ExecuteAsync(TableOperation.Retrieve(insertOrReplaceEntity.PartitionKey, insertOrReplaceEntity.RowKey));
DynamicTableEntity retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(insertOrReplaceEntity.Properties.Count, retrievedEntity.Properties.Count);
DynamicTableEntity replaceEntity = new DynamicTableEntity(insertOrReplaceEntity.PartitionKey, insertOrReplaceEntity.RowKey);
replaceEntity.Properties.Add("prop2", new EntityProperty("value2"));
TableBatchOperation batch2 = new TableBatchOperation();
batch2.InsertOrReplace(replaceEntity);
await currentTable.ExecuteBatchAsync(batch2);
// Retrieve Entity & Verify Contents
result = await currentTable.ExecuteAsync(TableOperation.Retrieve(insertOrReplaceEntity.PartitionKey, insertOrReplaceEntity.RowKey));
retrievedEntity = result.Result as DynamicTableEntity;
Assert.IsNotNull(retrievedEntity);
Assert.AreEqual(1, retrievedEntity.Properties.Count);
Assert.AreEqual(replaceEntity.Properties["prop2"], retrievedEntity.Properties["prop2"]);
}
private async Task SyncTableSince(CloudTable table, DateTimeOffset timestamp)
{
var query = new TableQuery<DynamicTableEntity>
{
FilterString = TableQuery.GenerateFilterConditionForDate("Timestamp", "gt", timestamp),
};
IList<DynamicTableEntity> allFuckingEntities = await table.ExecuteQueryAsync(
query, _token, list => Console.WriteLine("loaded {0} rows", list.Count));
CloudTable dstTable = _dstClient.GetTableReference(table.Name);
await dstTable.CreateIfNotExistsAsync();
int n = 0;
DateTimeOffset maxSourceTs = timestamp;
foreach (var batch1 in allFuckingEntities.GroupBy(x => x.PartitionKey))
{
if (_token.IsCancellationRequested)
return;
foreach (var batch2 in batch1.Batch(100))
{
if (_token.IsCancellationRequested)
return;
var op = new TableBatchOperation();
foreach (DynamicTableEntity entity in batch2)
{
op.InsertOrReplace(entity);
if (entity.Timestamp > maxSourceTs)
{
maxSourceTs = entity.Timestamp;
}
}
await dstTable.ExecuteBatchAsync(op, _token);
n += Math.Min(op.Count, 100);
Console.WriteLine("sent {0} rows", n);
}
}
_timestamps[table] = maxSourceTs;
}
public async Task TableBatchOnSecondaryAsync()
{
AssertSecondaryEndpoint();
CloudTable table = GenerateCloudTableClient().GetTableReference(GenerateRandomTableName());
TableRequestOptions options = new TableRequestOptions()
{
LocationMode = LocationMode.SecondaryOnly,
RetryPolicy = new NoRetry(),
};
TableBatchOperation batch = new TableBatchOperation();
batch.Retrieve("PartitionKey", "RowKey");
OperationContext context = new OperationContext();
await table.ExecuteBatchAsync(batch, options, context);
Assert.AreEqual(StorageLocation.Secondary, context.LastResult.TargetLocation);
batch = new TableBatchOperation();
batch.Insert(new DynamicTableEntity("PartitionKey", "RowKey"));
StorageException e = await TestHelper.ExpectedExceptionAsync<StorageException>(
async () => await table.ExecuteBatchAsync(batch, options, null),
"Batch operations other than retrieve should not be sent to secondary");
Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message);
batch = new TableBatchOperation();
batch.InsertOrMerge(new DynamicTableEntity("PartitionKey", "RowKey"));
e = await TestHelper.ExpectedExceptionAsync<StorageException>(
async () => await table.ExecuteBatchAsync(batch, options, null),
"Batch operations other than retrieve should not be sent to secondary");
Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message);
batch = new TableBatchOperation();
batch.InsertOrReplace(new DynamicTableEntity("PartitionKey", "RowKey"));
e = await TestHelper.ExpectedExceptionAsync<StorageException>(
async () => await table.ExecuteBatchAsync(batch, options, null),
"Batch operations other than retrieve should not be sent to secondary");
Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message);
batch = new TableBatchOperation();
batch.Merge(new DynamicTableEntity("PartitionKey", "RowKey") { ETag = "*" });
e = await TestHelper.ExpectedExceptionAsync<StorageException>(
async () => await table.ExecuteBatchAsync(batch, options, null),
"Batch operations other than retrieve should not be sent to secondary");
Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message);
batch = new TableBatchOperation();
batch.Replace(new DynamicTableEntity("PartitionKey", "RowKey") { ETag = "*" });
e = await TestHelper.ExpectedExceptionAsync<StorageException>(
async () => await table.ExecuteBatchAsync(batch, options, null),
"Batch operations other than retrieve should not be sent to secondary");
Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message);
batch = new TableBatchOperation();
batch.Delete(new DynamicTableEntity("PartitionKey", "RowKey") { ETag = "*" });
e = await TestHelper.ExpectedExceptionAsync<StorageException>(
async () => await table.ExecuteBatchAsync(batch, options, null),
"Batch operations other than retrieve should not be sent to secondary");
Assert.AreEqual(SR.PrimaryOnlyCommand, e.Message);
}
public void Execute()
{
try
{
// Retrieve the storage account from the connection string.
CloudStorageAccount account;
if (CloudStorageAccount.TryParse(_connectionString, out account))
{
// Create the table client.
CloudTableClient client = account.CreateCloudTableClient();
// Create the table if it doesn't exist.
CloudTable table = client.GetTableReference("postalcodegb");
table.CreateIfNotExists();
// TO DO : relative path? ~ didnt work :(
using (var reader = new TextFieldParser(Path.Combine(_appDataFolder, "GB.txt")))
{
reader.TextFieldType = FieldType.Delimited;
reader.Delimiters = new string[] { "\t" };
string[] currentRow;
TableBatchOperation batchOp = new TableBatchOperation();
while (!reader.EndOfData)
{
while (batchOp.Count < 100)
{
currentRow = reader.ReadFields();
var postalCodeEntity = new PostalCodeEntity()
{
iso2 = currentRow[0],
postalCode = currentRow[1],
placeName = currentRow[2],
stateName = currentRow[3],
stateCode = currentRow[4],
countyName = currentRow[5],
countyCode = currentRow[6],
communityName = currentRow[7],
communityCode = currentRow[8],
latitude = string.IsNullOrEmpty(currentRow[9]) ? (float)0 : float.Parse(currentRow[9], CultureInfo.InvariantCulture.NumberFormat),
longitude = string.IsNullOrEmpty(currentRow[10]) ? (float)0 : float.Parse(currentRow[10], CultureInfo.InvariantCulture.NumberFormat),
accuracy = string.IsNullOrEmpty(currentRow[11]) ? (int)0 : int.Parse(currentRow[11], CultureInfo.InvariantCulture.NumberFormat)
};
batchOp.InsertOrReplace(postalCodeEntity);
}
table.ExecuteBatch(batchOp);
}
}
}
}
catch (Exception ex)
{
Log.Error(string.Format(CultureInfo.InvariantCulture, "Failed to load data into Azure Table Storage: {0}", ex.Message), ex);
}
}
private async Task DoTableBatchAllSupportedOperationsAsync(TablePayloadFormat format)
{
tableClient.DefaultRequestOptions.PayloadFormat = format;
TableBatchOperation batch = new TableBatchOperation();
string pk = Guid.NewGuid().ToString();
// insert
batch.Insert(GenerateRandomEntity(pk));
// delete
{
DynamicTableEntity entity = GenerateRandomEntity(pk);
await currentTable.ExecuteAsync(TableOperation.Insert(entity));
batch.Delete(entity);
}
// replace
{
DynamicTableEntity entity = GenerateRandomEntity(pk);
await currentTable.ExecuteAsync(TableOperation.Insert(entity));
batch.Replace(entity);
}
// insert or replace
{
DynamicTableEntity entity = GenerateRandomEntity(pk);
await currentTable.ExecuteAsync(TableOperation.Insert(entity));
batch.InsertOrReplace(entity);
}
// merge
{
DynamicTableEntity entity = GenerateRandomEntity(pk);
await currentTable.ExecuteAsync(TableOperation.Insert(entity));
batch.Merge(entity);
}
// insert or merge
{
DynamicTableEntity entity = GenerateRandomEntity(pk);
await currentTable.ExecuteAsync(TableOperation.Insert(entity));
batch.InsertOrMerge(entity);
}
IList<TableResult> results = await currentTable.ExecuteBatchAsync(batch);
Assert.AreEqual(results.Count, 6);
IEnumerator<TableResult> enumerator = results.GetEnumerator();
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.Created);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
enumerator.MoveNext();
Assert.AreEqual(enumerator.Current.HttpStatusCode, (int)HttpStatusCode.NoContent);
}
public static void SaveToStorage(IEnumerable<TemplateModel> templates)
{
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (TemplateModel template in templates)
{
batchOperation.InsertOrReplace(template);
}
StorageFactory.Instance.IpcAzureAppTenantStateTable.ExecuteBatch(batchOperation, tableReqOptions);
}
public void BatchOperationExceptionWhenUsingSmallerViewId()
{
long currentViewId = 100;
long badViewId = currentViewId - 1;
this.UpdateConfiguration(replicas, 0, false, currentViewId);
string jobType = "jobType-BatchOperationExceptionWhenUsingSmallerViewId";
string jobId = "jobId-BatchOperationExceptionWhenUsingSmallerViewId";
int count = 3; // number of operations in the batch _rtable_Operation
List<TableOperationType> opTypes = new List<TableOperationType>()
{
TableOperationType.Replace,
TableOperationType.InsertOrReplace,
TableOperationType.Delete,
};
//
// Insert
//
string jobIdTemplate = jobId + "-{0}";
string messageTemplate = "message-{0}";
string updatedMessageTemplate = "updated-" + messageTemplate;
string partitionKey = string.Empty;
//
// Insert entities
//
for (int i = 0; i < count; i++)
{
SampleRTableEntity originalEntity = new SampleRTableEntity(
jobType,
string.Format(jobIdTemplate, i),
string.Format(messageTemplate, i));
this.repTable.Execute(TableOperation.Insert(originalEntity));
partitionKey = originalEntity.PartitionKey;
}
//
// Retrieve entities and use them to create batchOperation to Replace or Delete
//
IEnumerable<SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey);
TableBatchOperation batchOperation = new TableBatchOperation();
int m = 0;
foreach (SampleRTableEntity entity in allEntities)
{
Console.WriteLine("{0}", entity.ToString());
Console.WriteLine("---------------------------------------");
if (opTypes[m] == TableOperationType.Replace)
{
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(updatedMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
batchOperation.Replace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.InsertOrReplace)
{
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(updatedMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
batchOperation.InsertOrReplace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.Delete)
{
entity.ETag = entity._rtable_Version.ToString();
batchOperation.Delete(entity);
}
else
{
throw new ArgumentException(
string.Format("opType={0} is NOT supported", opTypes[m]),
"opType");
}
m++;
}
//
// Call ModifyConfigurationBlob to change the viewId of the wrapper to an older value
//
Console.WriteLine("Changing the viewId to badViewId {0}", badViewId);
this.UpdateConfiguration(replicas, 0, false, badViewId);
//
// Execute Batch _rtable_Operation with bad viewId
//
Console.WriteLine("\nCalling BatchOperation with badViewId...");
try
{
this.repTable.ExecuteBatch(batchOperation);
}
catch (ReplicatedTableStaleViewException ex)
{
Console.WriteLine("Get this RTableStaleViewException: {0}", ex.Message);
Assert.IsTrue(ex.Message.Contains(string.Format("current _rtable_ViewId {0} is smaller than", badViewId)), "Got unexpected exception message");
}
}
private void PersistQueryTable(IEnumerable<TwitterSearchRow> queries)
{
var batchOperation = new TableBatchOperation();
foreach(var query in queries)
{
batchOperation.InsertOrReplace(query);
}
MetadataTable.ExecuteBatch(batchOperation);
}
public void Execute(TableBatchOperation batchOperation, TableEntity entity)
{
batchOperation.InsertOrReplace(entity);
}
/// <summary>
/// Insert a batch of ElevationDataSetTableEntity into the storage table
/// </summary>
/// <param name="table"></param>
/// <param name="entities"></param>
private static void InsertBatchOfElevationDataSetTableEntity(CloudTable table, List<ElevationDataSetTableEntity> entities)
{
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (ElevationDataSetTableEntity tableEntity in entities)
batchOperation.InsertOrReplace(tableEntity);
try
{
table.ExecuteBatch(batchOperation);
}
catch(Exception ex)
{
Console.WriteLine(ex.Message);
throw;
}
}
