public async Task ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> messages) { var batch = new TableBatchOperation(); foreach(var msg in messages) { var snap = JsonConvert.DeserializeObject<BusSnapshotInfo>(Encoding.UTF8.GetString(msg.GetBytes())); var entity = new DynamicTableEntity(snap.RouteShortName, snap.VehicleId.ToString()); entity.Properties.Add("RouteShortName", EntityProperty.GeneratePropertyForString(snap.RouteShortName)); entity.Properties.Add("VehicleId", EntityProperty.GeneratePropertyForInt(snap.VehicleId)); entity.Properties.Add("TripId", EntityProperty.GeneratePropertyForInt(snap.TripId)); entity.Properties.Add("Latitude", EntityProperty.GeneratePropertyForDouble(snap.Latitude)); entity.Properties.Add("Longitude", EntityProperty.GeneratePropertyForDouble(snap.Longitude)); entity.Properties.Add("DirectionOfTravel", EntityProperty.GeneratePropertyForString(snap.DirectionOfTravel.ToString())); entity.Properties.Add("NextStopId", EntityProperty.GeneratePropertyForInt(snap.NextStopId)); entity.Properties.Add("Timeliness", EntityProperty.GeneratePropertyForString(snap.Timeliness.ToString())); entity.Properties.Add("TimelinessOffset", EntityProperty.GeneratePropertyForInt(snap.TimelinessOffset)); entity.Properties.Add("Timestamp", EntityProperty.GeneratePropertyForDateTimeOffset(snap.Timestamp)); batch.Add(TableOperation.InsertOrReplace(entity)); } var tableClient = _account.CreateCloudTableClient(); var table = tableClient.GetTableReference("snapshots"); await table.CreateIfNotExistsAsync(); await table.ExecuteBatchAsync(batch); await context.CheckpointAsync(); }
public void Insert() { TableBatchOperation batchOperation = new TableBatchOperation(); Hashtable barcode = new Hashtable(); StringBuilder sb = new StringBuilder(); for (int i = id; i < id + 100 && i < Program._DATA_TABLE.Rows.Count; i++) { if (!barcode.ContainsKey(Program._DATA_TABLE.Rows[i]["Barcode"].ToString().Trim())) { try { sb.Append(Program._DATA_TABLE.Rows[i]["Barcode"].ToString().Trim() + "|"); ShopBarcodeEntity data = new ShopBarcodeEntity(Program._DATA_TABLE.Rows[i]["Shop"].ToString().Trim(), Program._DATA_TABLE.Rows[i]["Barcode"].ToString().Trim()); data.OrderNo = Program._DATA_TABLE.Rows[i]["BillNumber"].ToString().Trim(); data.Product = Program._DATA_TABLE.Rows[i]["Product"].ToString().Trim().PadLeft(8, '0'); data.Cost = double.Parse(Program._DATA_TABLE.Rows[i]["SellPrice"].ToString().Trim()); data.SellFinished = false; batchOperation.InsertOrMerge(data); barcode[Program._DATA_TABLE.Rows[i]["Barcode"].ToString().Trim()] = true; } catch { } } } try { Program._RECORD++; Console.WriteLine("Insert Record {0}-{1}\t\tTotal {2} Records", id + 1, id + 100, Program._RECORD*100); Program._CLOUD_TABLE.ExecuteBatch(batchOperation); } catch (Exception e) { Program.WriteErrorLog("Record " + (id + 1) + "-" + (id + 100) + " Error \n" + sb.ToString() + "\n" + e.Message + "\n" + e.StackTrace); } }
public List<Reply> GetReplyNotif(string userid) { TableQuery<ReplyNotificationEntifity> query = new TableQuery<ReplyNotificationEntifity>().Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, userid)); List<Reply> replies = new List<Reply>(); TableBatchOperation batchOperation = new TableBatchOperation(); int count = 0; // Print the fields for each customer. foreach (ReplyNotificationEntifity entity in _replyNotification.ExecuteQuery(query)) { replies.Add(JsonConvert.DeserializeObject<Reply>(entity.Content)); batchOperation.Add(TableOperation.Delete(entity)); count++; if((count % 100) == 0){ _replyNotification.ExecuteBatch(batchOperation); batchOperation = new TableBatchOperation(); count = 0; } } if (count > 0) { _replyNotification.ExecuteBatch(batchOperation); } return replies; }
static void CreateCustomerMetadata(CloudTableClient tableClient) { Console.WriteLine("Creating customers metadata..."); CloudTable customersMetadataTable = tableClient.GetTableReference("customersmetadata"); customersMetadataTable.CreateIfNotExists(); var msftAddress1 = new DictionaryTableEntity(); msftAddress1.PartitionKey = "MSFT"; msftAddress1.RowKey = "ADDRESS-" + Guid.NewGuid().ToString("N").ToUpper(); msftAddress1.Add("city", "Seattle"); msftAddress1.Add("street", "111 South Jackson"); var msftWebsite1 = new DictionaryTableEntity(); msftWebsite1.PartitionKey = "MSFT"; msftWebsite1.RowKey = "WEBSITE-" + Guid.NewGuid().ToString("N").ToUpper(); msftWebsite1.Add("url", "http://www.microsoft.com"); var msftWebsite2 = new DictionaryTableEntity(); msftWebsite2.PartitionKey = "MSFT"; msftWebsite2.RowKey = "WEBSITE-" + Guid.NewGuid().ToString("N").ToUpper(); msftWebsite2.Add("url", "http://www.windowsazure.com"); var batch = new TableBatchOperation(); batch.Add(TableOperation.Insert(msftAddress1)); batch.Add(TableOperation.Insert(msftWebsite1)); batch.Add(TableOperation.Insert(msftWebsite2)); customersMetadataTable.ExecuteBatch(batch); Console.WriteLine("Done. Press ENTER to read the customer metadata."); Console.ReadLine(); }
/// <summary> /// Emit a batch of log events, running to completion synchronously. /// </summary> /// <param name="events">The events to emit.</param> /// <remarks>Override either <see cref="PeriodicBatchingSink.EmitBatch"/> or <see cref="PeriodicBatchingSink.EmitBatchAsync"/>, /// not both.</remarks> protected override void EmitBatch(IEnumerable<LogEvent> events) { var operation = new TableBatchOperation(); var first = true; foreach (var logEvent in events) { if (first) { //check to make sure the partition key is not the same as the previous batch if (partitionKey != logEvent.Timestamp.Ticks) { batchRowId = 0; //the partitionkey has been reset partitionKey = logEvent.Timestamp.Ticks; //store the new partition key } first = false; } var logEventEntity = new LogEventEntity(logEvent, _formatProvider, partitionKey); logEventEntity.RowKey += "|" + batchRowId; operation.Add(TableOperation.Insert(logEventEntity)); batchRowId++; } _table.ExecuteBatch(operation); }
private void InsertBets() { if (this.Bets.Count == 0) { return; } var tableBets = this.CreateTableClient("Bet"); tableBets.CreateIfNotExists(); var batchOperation = new TableBatchOperation(); foreach (var kvp in this.Bets) { var bet = new BetEntity(); bet.TicketNumber = kvp.Key; bet.BetNumber = kvp.Value; bet.RaffleId = this.RaffleId; batchOperation.Insert(bet); } tableBets.ExecuteBatch(batchOperation); }
public long ClearTable() { long deletionCount = 0; // Construct the query operation for all customer entities where PartitionKey="Smith". var list = new List<string>(); list.Add("PartitionKey"); list.Add("RowKey"); TableQuery<SensorValueEntity> query = new TableQuery<SensorValueEntity>().Select(list).Take(100); var results = table.ExecuteQuery(query); if (results.Count() < 1) return deletionCount; foreach(var resultGroup in results.GroupBy(a => a.PartitionKey)) { TableBatchOperation batchOperation = new TableBatchOperation(); foreach (var result in resultGroup) { batchOperation.Delete(result); deletionCount++; } table.ExecuteBatch(batchOperation); } return deletionCount; }
async Task<CommandResult> DoInsert(CloudTable table, long n, Func<long, EntityNk[]> entityFactory) { var batchOperation = new TableBatchOperation(); foreach (var e in entityFactory(n)) { batchOperation.Insert(e); } var cresult = new CommandResult { Start = DateTime.UtcNow.Ticks }; var cbt = 0L; var context = GetOperationContext((t) => cbt = t); try { var results = await table.ExecuteBatchAsync(batchOperation, operationContext: context); cresult.Elapsed = cbt; } catch (Exception ex) { cresult.Elapsed = -1; Console.Error.WriteLine("Error DoInsert {0} {1}", n, ex.ToString()); } return cresult; }
public AzureTableContext(CloudTable table, IEncryptor encryptor) { _table = table; _encryptor = encryptor; _context = new TableBatchOperation(); _deleteBackupContext = new TableBatchOperation(); }
protected override void Append(LoggingEvent[] loggingEvents) { try { TableBatchOperation batchOperation = new TableBatchOperation(); base.Append(loggingEvents); foreach (var e in loggingEvents) { var logitem = new LogItem { Exception = e.GetExceptionString(), Level = e.Level.Name, LoggerName = e.LoggerName, Message = e.RenderedMessage, RoleInstance = RoleInfo }; batchOperation.Insert(logitem); } _log4NetTable.ExecuteBatch(batchOperation); } catch (Exception ex) { Trace.TraceError("AzureTableAppender Append " + ex.Message); } }
public void Insert() { TableBatchOperation batchOperation = new TableBatchOperation(); Hashtable bank = new Hashtable(); for (int i = id; i < id + 100 && i < Program._DATA_TABLE.Rows.Count; i++) { if (!bank.ContainsKey(Program._DATA_TABLE.Rows[i]["ID"].ToString().Trim())) { try { DynamicTableEntity data = new DynamicTableEntity(); data.PartitionKey = "88888888"; data.RowKey = Program._DATA_TABLE.Rows[i]["ID"].ToString().Trim().PadLeft(6, '0'); Dictionary<string, EntityProperty> properties = new Dictionary<string, EntityProperty>(); properties.Add("Bank", new EntityProperty(Program._DATA_TABLE.Rows[i]["Bank"].ToString().Trim())); properties.Add("BranchName", new EntityProperty(Program._DATA_TABLE.Rows[i]["BranchName"].ToString().ToLower().Trim())); properties.Add("AccountNumber", new EntityProperty(Program._DATA_TABLE.Rows[i]["AccountNumber"].ToString().Trim())); properties.Add("AccountName", new EntityProperty(Program._DATA_TABLE.Rows[i]["AccountName"].ToString().Trim())); properties.Add("AccountType", new EntityProperty(int.Parse(Program._DATA_TABLE.Rows[i]["AccountType"].ToString().Trim()))); //BankEntity data = new BankEntity("88888888", Program._DATA_TABLE.Rows[i]["ID"].ToString().Trim().PadLeft(6, '0')); //data.Bank = Program._DATA_TABLE.Rows[i]["Bank"].ToString().Trim(); //data.BranchName = Program._DATA_TABLE.Rows[i]["BranchName"].ToString().ToLower().Trim(); //data.AccountNumber = Program._DATA_TABLE.Rows[i]["AccountNumber"].ToString().Trim(); //data.AccountName = Program._DATA_TABLE.Rows[i]["AccountName"].ToString().Trim(); //data.AccountType = int.Parse(Program._DATA_TABLE.Rows[i]["AccountType"].ToString().Trim()); batchOperation.InsertOrMerge(data); recBank++; bank[Program._DATA_TABLE.Rows[i]["ID"].ToString().Trim()] = true; } catch { } } } try { if (Program._DATA_TABLE.Rows.Count > 0) { if (Program._UPDATE) { Program._RECORD++; Console.WriteLine("Update Record {0}-{1}\t\tTotal {2} Records", id + 1, id + 100, Program._RECORD * 100); Program._CLOUD_TABLE.ExecuteBatch(batchOperation); } else { Program._RECORD++; Console.WriteLine("Insert Record {0}-{1}\t\tTotal {2} Records", id + 1, id + 100, Program._RECORD * 100); Program._CLOUD_TABLE.ExecuteBatch(batchOperation); } } } catch (Exception e) { Program.WriteErrorLog("Record " + (id + 1) + "-" + (id + 100) + " Error \n" + e.Message + "\n" + e.StackTrace); } }
public async void Add(IList<ITableEntity> notes) { var batchOperation = new TableBatchOperation(); notes.ToList().ForEach(n => batchOperation.Insert(n)); await _table.ExecuteBatchAsync(batchOperation); }
public static bool InsertEntityToAzureTable(string tableName, List<CustomerEntity> entityList) { System.Diagnostics.Trace.TraceInformation(string.Format("Date = {0}, AzureTableOperation.InsertEntityToAzureTable2 is Called", DateTime.Now)); // Retrieve the storage account from the connection string. CloudStorageAccount storageAccount = CloudStorageAccount.Parse(GetStorageConnectionString()); // Create the table client. CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); // Create the CloudTable object that represents the "people" table. CloudTable table = tableClient.GetTableReference(tableName); // Create the batch operation. TableBatchOperation batchOperation = new TableBatchOperation(); // Add both customer entities to the batch insert operation. foreach (CustomerEntity entity in entityList) { batchOperation.Insert(entity); } // Execute the insert operation. IList<TableResult> resultList = table.ExecuteBatch(batchOperation); System.Diagnostics.Trace.TraceInformation(string.Format("Date = {0}, AzureTableOperation.InsertEntityToAzureTable2 insert entity to {1}: {2}", DateTime.Now, tableName, resultList.Count)); //TODO: how to determine whether the operation is successful return true; }
public async Task Persist(ProjectionMetaData metadata) { Trace.TraceInformation("Preparing batch for projection {0}.", metadata.ProjectionType); var batch = new TableBatchOperation(); var entity = new DictionaryTableEntity { PartitionKey = metadata.ProjectionType, RowKey = Handler.Current() }; entity.Add("ProjectionHash", metadata.ProjectionHash); batch.Add(TableOperation.InsertOrReplace(entity)); Trace.TraceInformation("Executing batch for projection {0}.", metadata.ProjectionType); await _table.ExecuteBatchAsync(batch).ContinueWith(r => { Trace.TraceInformation("Batch for projection {0} complete {1} exceptions.", metadata.ProjectionType, r.Exception != null ? "with" : "without"); if (r.Exception != null) { r.Exception.Handle(exception => { Trace.TraceError(exception.ToString()); return true; }); } }); }
private TableBatchOperation ComposeBatch(IEnumerable<LogEvent> events) { var batch = new TableBatchOperation(); foreach (LogEvent e in events) { var row = new ElasticTableEntity { PartitionKey = e.EventTime.ToString("yy-MM-dd"), RowKey = e.EventTime.ToString("HH-mm-ss-fff") }; row.Add("source", e.SourceName); row.Add("severity", e.Severity); row.Add("message", e.FormattedMessage); row.Add("error", e.ErrorException == null ? string.Empty : e.ErrorException.ToString()); if (e.Properties != null) { foreach (var p in e.Properties) { if (p.Key == LogEvent.ErrorPropertyName) continue; row.Add(p.Key, p.Value); } } batch.Insert(row); } return batch.Count > 0 ? batch : null; }
public void TableBatchAddNullShouldThrow() { TableBatchOperation batch = new TableBatchOperation(); try { batch.Add(null); Assert.Fail(); } catch (ArgumentNullException) { // no op } catch (Exception) { Assert.Fail(); } try { batch.Insert(0, null); Assert.Fail(); } catch (ArgumentNullException) { // no op } catch (Exception) { Assert.Fail(); } }
public ActionResult DeleteConfirmed(string partitionKey) { // Delete all rows for this mailing list, that is, // Subscriber rows as well as MailingList rows. // Therefore, no need to specify row key. var query = new TableQuery<MailingList>().Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey)); var listRows = mailingListTable.ExecuteQuery(query).ToList(); var batchOperation = new TableBatchOperation(); int itemsInBatch = 0; foreach (MailingList listRow in listRows) { batchOperation.Delete(listRow); itemsInBatch++; if (itemsInBatch == 100) { mailingListTable.ExecuteBatch(batchOperation); itemsInBatch = 0; batchOperation = new TableBatchOperation(); } } if (itemsInBatch > 0) { mailingListTable.ExecuteBatch(batchOperation); } return RedirectToAction("Index"); }
void ClearTable(CloudTable table) { var deviceIds = _deviceService.GetDeviceIds(); foreach (var partitionKey in deviceIds) { TableBatchOperation batchDelete = new TableBatchOperation(); // gets all the entities in the table for this partition key string partitionCondition = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey); List<DynamicTableEntity> entities = table.ExecuteQuery(new TableQuery().Where(partitionCondition)).ToList(); entities.ForEach(e => { batchDelete.Add(TableOperation.Delete(e)); // Azure has a limit on batch operations if (batchDelete.Count == 100) { table.ExecuteBatch(batchDelete); batchDelete = new TableBatchOperation(); } }); // flush out whatever is left if (batchDelete.Count > 0) { table.ExecuteBatch(batchDelete); } } }
public void AddEntryToTable(string tableName) { CloudTable table = cloudTableClient.GetTableReference(tableName); table.CreateIfNotExists(); TableBatchOperation batch = new TableBatchOperation(); batch.Add(TableOperation.Insert(new TableEntity { RowKey = "Abhishek", PartitionKey = "Kolkata" })); batch.Add(TableOperation.Insert(new TableEntity { RowKey = "Abhijit", PartitionKey = "Kolkata" })); table.ExecuteBatch(batch); }
public async Task AddVersion(OwnershipRegistration registration, OwnershipOwner owner, string version) { CloudTableClient client = _account.CreateCloudTableClient(); CloudTable table = client.GetTableReference(OwnershipTableName); TableBatchOperation batch = new TableBatchOperation(); batch.InsertOrReplace(new TypedEntity(registration.GetKey(), owner.GetKey(), OwnerType)); batch.InsertOrReplace(new TypedEntity(registration.GetKey(), version, PackageType)); await table.ExecuteBatchAsync(batch); }
void PostProcess(TableBatchOperation batchOperation) { foreach (var result in Table.ExecuteBatch(batchOperation)) { if (result.Result == null) { throw new Exception("Error saving"); } } }
public void Insert() { TableBatchOperation batchOperation = new TableBatchOperation(); Hashtable TransferD = new Hashtable(); for (int i = id; i < id + 100 && i < Program._DATA_TABLE.Rows.Count; i++) { string Rowkey = (Program._DATA_TABLE.Rows[i]["ID"].ToString().Trim() + "-" + Program._DATA_TABLE.Rows[i]["SellNumber"].ToString().Trim()); if (!TransferD.ContainsKey(Rowkey)) { try { DynamicTableEntity data = new DynamicTableEntity(); data.PartitionKey = "88888888"; data.RowKey = Rowkey; Dictionary<string, EntityProperty> properties = new Dictionary<string, EntityProperty>(); properties.Add("SellNumber", new EntityProperty(Program._DATA_TABLE.Rows[i]["SellNumber"].ToString().Trim().PadLeft(8, '0'))); properties.Add("ReceiveMoney", new EntityProperty(double.Parse(Program._DATA_TABLE.Rows[i]["ReceiveMoney"].ToString().ToLower().Trim()))); //BankTransferDetailEntity data = new BankTransferDetailEntity("88888888", Rowkey); //data.SellNumber = Program._DATA_TABLE.Rows[i]["SellNumber"].ToString().Trim().PadLeft(8, '0'); //data.ReceiveMoney = double.Parse(Program._DATA_TABLE.Rows[i]["ReceiveMoney"].ToString().ToLower().Trim()); batchOperation.InsertOrMerge(data); recTransferD++; TransferD[Rowkey] = true; } catch { } } } try { if (Program._DATA_TABLE.Rows.Count > 0) { if (Program._UPDATE) { Program._RECORD++; Console.WriteLine("Update Record {0}-{1}\t\tTotal {2} Records", id + 1, id + 100, Program._RECORD * 100); Program._CLOUD_TABLE.ExecuteBatch(batchOperation); } else { Program._RECORD++; Console.WriteLine("Insert Record {0}-{1}\t\tTotal {2} Records", id + 1, id + 100, Program._RECORD * 100); Program._CLOUD_TABLE.ExecuteBatch(batchOperation); } } } catch (Exception e) { Program.WriteErrorLog("Record " + (id + 1) + "-" + (id + 100) + " Error \n" + e.Message + "\n" + e.StackTrace); } }
public TableBatchOperationAdapter( Func<TableBatchOperation, Task<IList<TableResult>>> executeBatch, ITableEntityConverter tableEntityConverter) { Require.NotNull(executeBatch, "executeBatch"); Require.NotNull(tableEntityConverter, "tableEntityConverter"); m_executeBatch = executeBatch; m_tableEntityConverter = tableEntityConverter; m_batch = new TableBatchOperation(); }
public void SaveToStorageInBatch(List<Log> list) { var batchOperation = new TableBatchOperation(); foreach (var log in list) { var entity = GetLogTableEntity(log); batchOperation.Insert(entity); } var result = _table.ExecuteBatch(batchOperation); }
public ActionResult About() { var storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); var client = storageAccount.CreateCloudTableClient(); var stebraTable = client.GetTableReference("StebraList"); if (!stebraTable.Exists()) { stebraTable.Create(); } TableOperation retrieveOperation = TableOperation.Retrieve<StebraEntity>("WeAreStebra", "ListItem3"); // Execute the operation. TableResult retrievedResult = stebraTable.Execute(retrieveOperation); StebraEntity result = (StebraEntity)retrievedResult.Result; if (result != null) { result.ImageUrl = "http://cdn.buzznet.com/assets/users16/crizz/default/funny-pictures-thriller-kitten-impresses--large-msg-121404159787.jpg"; //var awesomeStebra = new StebraEntity("WeAreStebra", "ListItem2"); //awesomeStebra.ImageUrl = "http://rubmint.com/wp-content/plugins/wp-o-matic/cache/6cb1b_funny-pictures-colur-blind-kitteh-finded-yew-a-pumikin.jpg"; //var coolStebra = new StebraEntity("WeAreStebra", "ListItem2"); //coolStebra.ImageUrl = "http://rubmint.com/wp-content/plugins/wp-o-matic/cache/6cb1b_funny-pictures-colur-blind-kitteh-finded-yew-a-pumikin.jpg"; //var batchOperation = new TableBatchOperation(); TableOperation batchOperation = TableOperation.InsertOrReplace(result); //batchOperation.Insert(awesomeStebra); //batchOperation.Insert(coolStebra); //stebraTable.ExecuteBatch(batchOperation); stebraTable.Execute(batchOperation); } else { var awesomeStebra = new StebraEntity("WeAreStebra", "ListItem3"); awesomeStebra.ImageUrl = "http://rubmint.com/wp-content/plugins/wp-o-matic/cache/6cb1b_funny-pictures-colur-blind-kitteh-finded-yew-a-pumikin.jpg"; var batchOperation = new TableBatchOperation(); batchOperation.Insert(awesomeStebra); stebraTable.ExecuteBatch(batchOperation); } return View(); }
public IEnumerable<Usuarios> GetCreataTableStorage() { CloudTable table = getTable(); table.CreateIfNotExists(); TableBatchOperation batch = new TableBatchOperation(); batch.InsertOrMerge(new Usuarios("usuarios", "759990") { Nome = "Christiano Donke", Email = "*****@*****.**" }); batch.InsertOrMerge(new Usuarios("usuarios", "677040") { Nome = "Alexandre Campos", Email = "*****@*****.**" }); table.ExecuteBatch(batch); return this.queryAzureStorage(); }
public void Delete() { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString); CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); CloudTable table = tableClient.GetTableReference(tableName); var entries = table.CreateQuery<EventTableEntity>().Where(e => e.PartitionKey == Id).ToArray(); var delete = new TableBatchOperation(); foreach (var eventTableEntity in entries) { delete.Add(TableOperation.Delete(eventTableEntity)); } table.ExecuteBatch(delete); }
public async Task Save(Page[] pages) { var table = await GetTable(); var batchInsertOperation = new TableBatchOperation(); foreach (var page in pages) { batchInsertOperation.Add(TableOperation.Insert(ToPageTableEntity(page))); } await table.ExecuteBatchAsync(batchInsertOperation); }
private static async Task BackupPostCollection(DocumentCollection dc, DocumentClient client) { Trace.TraceInformation("Collection '{0}' start. Time: '{1}'", dc.Id, DateTime.Now.ToString(CultureInfo.CurrentCulture)); try { var ds = from d in client.CreateDocumentQuery<PostMessage>(dc.DocumentsLink) where d.Type == "Post" select d; TableBatchOperation batchOperation = new TableBatchOperation(); List<dynamic> docList = new List<dynamic>(); foreach (var d in ds) { TablePost c = ModelService.TablePostData(d); batchOperation.Insert(c); docList.Add(d); if (batchOperation.Count == 100) { var operation = batchOperation; var res = await _retryPolicy.ExecuteAsync( () => _table.ExecuteBatchAsync(operation)); batchOperation = new TableBatchOperation(); if (res.Count == operation.Count) { await _iDbService.DocumentDb.BatchDelete(dc, docList); docList = new List<dynamic>(); Trace.TraceInformation("inserted"); } } } if (batchOperation.Count > 0) { var operation = batchOperation; var res = await _retryPolicy.ExecuteAsync( () => _table.ExecuteBatchAsync(operation)); if (res.Count == operation.Count) { await _iDbService.DocumentDb.BatchDelete(dc, docList); Trace.TraceInformation("inserted"); } } } catch (Exception e) { Trace.TraceError("Error in BackupCollection " + e.Message); } }
static TableBatchOperation SaveBootstrap(IEnumerable<SearchModel> searchModel) { var batchOperation = new TableBatchOperation(); foreach (var model in searchModel) { if (string.IsNullOrWhiteSpace(model.Url) || string.IsNullOrWhiteSpace(model.PartitionKey)) { throw new InvalidOperationException("We require a partition key, and row key"); } batchOperation.InsertOrReplace(model); } return batchOperation; }
public static void CreateTable(string connectionString, string tableName, IReadOnlyDictionary <string, object>[] data) { var table = GetTable(connectionString, tableName); table.CreateIfNotExists(); TableBatchOperation batch = new TableBatchOperation(); foreach (var entity in data) { batch.Insert(new DictionaryTableEntity(Guid.NewGuid().ToString(), entity)); if (batch.Count >= 100) { table.ExecuteBatch(batch); batch.Clear(); } } if (batch.Count > 0) { table.ExecuteBatch(batch); } }
/// <summary> /// Demonstrate inserting of a large batch of entities. Some considerations for batch operations: /// 1. You can perform updates, deletes, and inserts in the same single batch operation. /// 2. A single batch operation can include up to 100 entities. /// 3. All entities in a single batch operation must have the same partition key. /// 4. While it is possible to perform a query as a batch operation, it must be the only operation in the batch. /// 5. Batch size must be <= 4MB /// </summary> /// <param name="table">Sample table name</param> private static async Task BatchInsertOfCustomerEntitiesAsync(CloudTable table) { // Create the batch operation. TableBatchOperation batchOperation = new TableBatchOperation(); // The following code generates test data for use during the query samples. for (int i = 0; i < 100; i++) { batchOperation.InsertOrMerge(new CustomerEntity("Smith", string.Format("{0}", i.ToString("D4"))) { Email = string.Format("{0}@contoso.com", i.ToString("D4")), PhoneNumber = string.Format("425-555-{0}", i.ToString("D4")) }); } // Execute the batch operation. IList <TableResult> results = await table.ExecuteBatchAsync(batchOperation); foreach (var res in results) { var customerInserted = res.Result as CustomerEntity; Console.WriteLine("Inserted entity with\t Etag = {0} and PartitionKey = {1}, RowKey = {2}", customerInserted.ETag, customerInserted.PartitionKey, customerInserted.RowKey); } }
public void TableBatchOver4MBShouldThrow() { TableBatchOperation batch = new TableBatchOperation(); string pk = Guid.NewGuid().ToString(); for (int m = 0; m < 65; m++) { DynamicReplicatedTableEntity ent = GenerateRandomEnitity(pk); // Maximum Entity size is 64KB ent.Properties.Add("binary", EntityProperty.GeneratePropertyForByteArray(new byte[64 * 1024])); batch.Insert(ent); } OperationContext opContext = new OperationContext(); try { this.repTable.ExecuteBatch(batch, null, opContext); Assert.Fail(); } catch (StorageException) { // // ###XXX: NEW SOURCE CODES: The commented out codes were the original codes // //TestHelper.ValidateResponse(opContext, 1, (int)HttpStatusCode.BadRequest, new string[] { "ContentLengthExceeded" }, "The content length for the requested operation has exceeded the limit (4MB)."); TestHelper.ValidateResponse( opContext, 2, (int)HttpStatusCode.RequestEntityTooLarge, new string[] { "RequestBodyTooLarge" }, "The request body is too large and exceeds the maximum permissible limit" ); } }
public void TableBatchWithPropertyOver255CharsShouldThrow() { TableBatchOperation batch = new TableBatchOperation(); string pk = Guid.NewGuid().ToString(); string propName = new string('a', 256); // propName has 256 chars DynamicReplicatedTableEntity ent = new DynamicReplicatedTableEntity("foo", "bar"); ent.Properties.Add(propName, new EntityProperty("propbar")); batch.Insert(ent); OperationContext opContext = new OperationContext(); try { this.repTable.ExecuteBatch(batch, null, opContext); Assert.Fail(); } catch (StorageException) { TestHelper.ValidateResponse(opContext, 2, (int)HttpStatusCode.BadRequest, new string[] { "PropertyNameTooLong" }, "The property name exceeds the maximum allowed length (255)."); } }
private static TableBatchOperation GetBatchOperation(IEnumerable <CloudEventEntry> entries) { const int MaxBatchSizeInBytes = 4000000; var batchOperation = new TableBatchOperation(); int approximateBatchSize = 0; foreach (var item in entries) { var entity = item.CreateTableEntity(); approximateBatchSize += GetApproximateEntitySize(entity); if (approximateBatchSize >= MaxBatchSizeInBytes) { break; } batchOperation.InsertOrReplace(entity); } return(batchOperation); }
/// <summary> /// Delete Entities /// </summary> /// <param name="entities">Entities</param> /// <returns>Table Results</returns> public virtual async Task <IEnumerable <TableResult> > Delete(IEnumerable <ITableEntity> entities) { if (null == entities) { throw new ArgumentNullException("entities"); } if (!entities.Any()) { return(null); } var result = new List <TableResult>(); foreach (var batch in this.Batch(entities)) { var batchOperation = new TableBatchOperation(); batch.ToList().ForEach(e => batchOperation.Delete(e)); var r = await this.reference.ExecuteBatchAsync(batchOperation).ConfigureAwait(false); result.AddRange(r); } return(result); }
public static TableBatchOperation CreateScheduleEntityForAvailabilitySet(IList <VirtualMachineCrawlerResponse> filteredVmSet, int schedulerFrequency, List <string> azureFiOperationList, bool domainFlage) { var tableBatchOperation = new TableBatchOperation(); var random = new Random(); var randomExecutionDateTime = DateTime.UtcNow.AddMinutes(random.Next(1, schedulerFrequency)); var sessionId = Guid.NewGuid().ToString(); foreach (var item in filteredVmSet) { if (item == null) { continue; } string fiOperation = string.Empty; var actionType = GetActionType(item.RowKey, azureFiOperationList, item.State, out fiOperation); tableBatchOperation.InsertOrMerge(RuleEngineHelper.ConvertToScheduledRuleEntityForAvailabilitySet(item, sessionId, actionType, fiOperation, randomExecutionDateTime, domainFlage)); } return(tableBatchOperation); }
private async Task <IList <TableResult> > ExecuteBatchSafeAsync(TableBatchOperation batch) { try { return(await this.ExecuteBatchAsync(batch).ConfigureAwait(false)); } catch (StorageException ex) { if (ex.RequestInformation == null || ex.RequestInformation.HttpStatusCode != 404 || // Not Found ex.RequestInformation.ExtendedErrorInformation == null || ex.RequestInformation.ExtendedErrorInformation.ErrorCode != "TableNotFound") { throw; } } if (await this.EnsureTableExistsAsync().ConfigureAwait(false)) { return(await this.ExecuteBatchAsync(batch).ConfigureAwait(false)); } return(null); }
private static async void BatchDelete <T>(TraceWriter log, CloudTable table, string partitionKey, List <T> items) where T : ITableEntity { TableBatchOperation operation = new TableBatchOperation(); foreach (var item in items) { operation.Delete(item); if (operation.Count >= 100) { await table.ExecuteBatchAsync(operation); operation = new TableBatchOperation(); log.Info($"Executing deletion batch on 100 items for {partitionKey}"); } } // Remove any remaining items, outside of the 100s batches. if (operation.Count > 0) { log.Info($"Executing deletion batch on {operation.Count} items for {partitionKey}"); await table.ExecuteBatchAsync(operation); } }
public static async Task SaveData([ActivityTrigger] DurableActivityContext context) { // retrieves a list of books from the Orchestrator function var books = context.GetInput <List <Book> >(); // create a table storage client var client = account.CreateCloudTableClient(); var table = client.GetTableReference("Books"); await table.CreateIfNotExistsAsync(); TableBatchOperation tableBatchOperations = new TableBatchOperation(); for (int i = 0; i < books.Count; i++) { tableBatchOperations.Add(TableOperation.InsertOrMerge( new BookRepository(books[i].Id) { Name = books[i].Name })); } await table.ExecuteBatchAsync(tableBatchOperations); }
protected async void Button4_Click(object sender, EventArgs e) { CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount; CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); CloudTable table = tableClient.GetTableReference("Urunler"); TableBatchOperation batchOperation = new TableBatchOperation(); for (int i = 0; i < 100; i++) { batchOperation.InsertOrMerge(new Urun() { PartitionKey = "Musteri" + i.ToString(), RowKey = (new Random().Next(1, 100)).ToString(), Adi = "Deneme", Aciklama = "Açıklama" }); } IList <TableResult> results = await table.ExecuteBatchAsync(batchOperation); foreach (var res in results) { var eklenenUrun = res.Result as Urun; } }
public async Task WriteDataBuffer(Table table, IEnumerable <object[]> buffer, string targetTableName, CancellationToken cancellationToken) { var connection = GetCloudTableClient(); var cloudTable = connection.GetTableReference(targetTableName); // Create the batch operation. var batchOperation = new TableBatchOperation(); var partitionKey = table.GetDeltaColumnOrdinal(TableColumn.EDeltaType.AzurePartitionKey); var rowKey = table.GetDeltaColumnOrdinal(TableColumn.EDeltaType.AzureRowKey); var surrogateKey = table.GetDeltaColumnOrdinal(TableColumn.EDeltaType.SurrogateKey); foreach (var row in buffer) { var properties = new Dictionary <string, EntityProperty>(); for (var i = 0; i < table.Columns.Count; i++) { if (table.Columns[i].DeltaType != TableColumn.EDeltaType.AzureRowKey && table.Columns[i].DeltaType != TableColumn.EDeltaType.AzurePartitionKey && table.Columns[i].DeltaType != TableColumn.EDeltaType.TimeStamp) { var value = row[i]; if (value == DBNull.Value) { value = null; } properties.Add(table.Columns[i].Name, NewEntityProperty(table.Columns[i].DataType, value)); } } var partionKeyValue = partitionKey >= 0 ? row[partitionKey] : "default"; var rowKeyValue = rowKey >= 0 ? row[rowKey] : surrogateKey >= 0 ? ((long)row[surrogateKey]).ToString("D20") : Guid.NewGuid().ToString(); var entity = new DynamicTableEntity(partionKeyValue.ToString(), rowKeyValue.ToString(), "*", properties); batchOperation.Insert(entity); } await cloudTable.ExecuteBatchAsync(batchOperation, null, null, cancellationToken); }
public async Task <bool> AddSubscriptions(IEnumerable <UserSubscription> subscriptionsToAdd) { CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); CloudTable userSubscriptionsTable = tableClient.GetTableReference("userSubscriptions"); var tableExists = await userSubscriptionsTable.ExistsAsync(); if (!tableExists) { await userSubscriptionsTable.CreateIfNotExistsAsync(); } TableBatchOperation batchOperation = new TableBatchOperation(); foreach (var subscription in subscriptionsToAdd) { UserSubscriptionEntity userSubscriptionEntity = new UserSubscriptionEntity(subscription.UserId, subscription.FriendId); batchOperation.InsertOrReplace(userSubscriptionEntity); } await userSubscriptionsTable.ExecuteBatchAsync(batchOperation); return(true); }
/// <summary> /// Make azure call to write to Table /// </summary> /// <param name="count">no of calls to be made</param> public static void MakeAzureCallToWriteTableWithSdk(int count) { // Retrieve storage account from connection string. CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString")); // Create the table client. CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); // Create the table if it doesn't exist. CloudTable table = tableClient.GetTableReference("people"); table.CreateIfNotExists(); // Create the batch operation. TableBatchOperation batchOperation = new TableBatchOperation(); // Create a customer entity and add it to the table. CustomerEntity customer1 = new CustomerEntity("Smith", "Jeff" + DateTime.UtcNow.Ticks); customer1.Email = "*****@*****.**"; customer1.PhoneNumber = "425-555-0104"; // Create another customer entity and add it to the table. CustomerEntity customer2 = new CustomerEntity("Smith", "Ben" + DateTime.UtcNow.Ticks); customer2.Email = "*****@*****.**"; customer2.PhoneNumber = "425-555-0102"; // Add both customer entities to the batch insert operation. batchOperation.Insert(customer1); batchOperation.Insert(customer2); // Execute the batch operation. table.ExecuteBatch(batchOperation); }
private async Task <List <TableResult> > ExecOrThrowAsync(CloudTable table, TableBatchOperation op) { try { return((await table.ExecuteBatchAsync(op)).ToList()); } catch (AzSE ex) { switch (ex.RequestInformation.HttpStatusCode) { case (int)HttpStatusCode.Conflict: throw new MeSE(ErrorCode.DuplicateKey, ex); case (int)HttpStatusCode.PreconditionFailed: throw new MeSE(ErrorCode.PreconditionFailed, ex); case (int)HttpStatusCode.BadRequest: throw new MeSE(ErrorCode.BadRequest, ex); default: throw new MeSE(ex.Message, ex); } } }
private static void doBatchOperation(CloudTable table) { // Create the batch operation. TableBatchOperation batchOperation = new TableBatchOperation(); // Create a customer entity and add it to the table. CustomerEntity customer1 = new CustomerEntity("Smith", "Jeff"); customer1.Email = "*****@*****.**"; customer1.PhoneNumber = "425-555-0104"; // Create another customer entity and add it to the table. CustomerEntity customer2 = new CustomerEntity("Smith", "Ben"); customer2.Email = "*****@*****.**"; customer2.PhoneNumber = "425-555-0102"; // Add both customer entities to the batch insert operation. batchOperation.Insert(customer1); batchOperation.Insert(customer2); // Execute the batch operation. table.ExecuteBatch(batchOperation); }
async Task <string> UploadHistoryBatch( string instanceId, string executionId, TableBatchOperation historyEventBatch, StringBuilder historyEventNamesBuffer, int numberOfTotalEvents, string eTagValue) { // Adding / updating sentinel entity DynamicTableEntity sentinelEntity = new DynamicTableEntity(instanceId, SentinelRowKey) { Properties = { ["ExecutionId"] = new EntityProperty(executionId), } }; if (!string.IsNullOrEmpty(eTagValue)) { sentinelEntity.ETag = eTagValue; historyEventBatch.Replace(sentinelEntity); } else { historyEventBatch.InsertOrReplace(sentinelEntity); } Stopwatch stopwatch = Stopwatch.StartNew(); IList <TableResult> tableResultList; try { tableResultList = await this.historyTable.ExecuteBatchAsync( historyEventBatch, this.StorageTableRequestOptions, null); } catch (StorageException ex) { if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed) { AnalyticsEventSource.Log.SplitBrainDetected( this.storageAccountName, this.taskHubName, instanceId, executionId, historyEventBatch.Count, numberOfTotalEvents, historyEventNamesBuffer.ToString(0, historyEventNamesBuffer.Length - 1), // remove trailing comma stopwatch.ElapsedMilliseconds, eTagValue); } throw; } this.stats.StorageRequests.Increment(); this.stats.TableEntitiesWritten.Increment(historyEventBatch.Count); if (tableResultList != null) { for (int i = tableResultList.Count - 1; i >= 0; i--) { if (((DynamicTableEntity)tableResultList[i].Result).RowKey == SentinelRowKey) { eTagValue = tableResultList[i].Etag; break; } } } AnalyticsEventSource.Log.AppendedInstanceHistory( this.storageAccountName, this.taskHubName, instanceId, executionId, historyEventBatch.Count, numberOfTotalEvents, historyEventNamesBuffer.ToString(0, historyEventNamesBuffer.Length - 1), // remove trailing comma stopwatch.ElapsedMilliseconds, this.GetETagValue(instanceId)); return(eTagValue); }
private static void GenerateCustomerRecords(TextWriter log) { if (!ConfirmOperation("YOUR STORAGE TABLE WILL BE RECREATED! ALL EXISTING DATA WILL BE LOST! Are you sure?", log)) { return; } var cloudTableClient = CloudStorageAccount.CreateCloudTableClient(); var cloudTable = cloudTableClient.GetTableReference(DemoSettings.Storage.CustomerTableName); if (cloudTable.DeleteIfExists()) { log.WriteLine("DELETING PREEXISTING TABLE '{0}'", cloudTable.Name); } while (true) { try { cloudTable.CreateIfNotExists(); break; } catch { Task.Delay(TimeSpan.FromSeconds(5)); } log.WriteLine("RETRY TABLE CREATION"); } int count = 1; const int totalRecords = 1000000; string[] firstNames = File.ReadAllLines("CSV_Database_of_First_Names.csv"); string[] lastNames = File.ReadAllLines("CSV_Database_of_Last_Names.csv"); var rand = new Random(DateTime.UtcNow.Millisecond); var batches = new Dictionary <string, TableBatchOperation>(); var tasks = new List <Task>(); while (count <= totalRecords) { count++; var company = "Company " + rand.Next(1, 11); lock (BatchLock) { if (!batches.ContainsKey(company)) { batches.Add(company, new TableBatchOperation()); } } TableBatchOperation tableBatchOperation; lock (BatchLock) { var customer = new Customer(company, count.ToString(CultureInfo.InvariantCulture)) { Value = (rand.NextDouble() - 0.5) * 99999.0, ContractDate = DateTime.Now, Name = firstNames[rand.Next(0, firstNames.Length)] + " " + lastNames[rand.Next(0, lastNames.Length)] }; tableBatchOperation = batches[company]; tableBatchOperation.Insert(customer); if (tableBatchOperation.Count < Batchsize) { continue; } batches[company] = new TableBatchOperation(); } tasks.Add(new Task(() => cloudTable.ExecuteBatch(tableBatchOperation))); } var remainingTasks = batches .Where(keyValuePair => keyValuePair.Value.Count > 0) .Select(keyValuePair => new Task(() => cloudTable.ExecuteBatch(keyValuePair.Value))); tasks.AddRange(remainingTasks); // No tasks are running before this line. The TaskRunner will throttle to a specific # tasks const int tasksInParallel = 10; var taskRunner = new TaskRunner(tasks, tasksInParallel); taskRunner.TaskCompleted += (o, e) => log.WriteLine("Tasks running: {0}. Average time: {1}. Tasks completed {2} . Total Time {3}.", e.TasksInParallel, CalculateAverage(e.TaskTimeTotal, e.TasksCompleted).ToString("g"), e.TasksCompleted, e.TaskTimeTotal.ToString("g")); // Run all tasks taskRunner.WaitAll(); }
/// <inheritdoc /> public override async Task UpdateStateAsync(OrchestrationRuntimeState runtimeState, string instanceId, string executionId) { IList <HistoryEvent> newEvents = runtimeState.NewEvents; IList <HistoryEvent> allEvents = runtimeState.Events; var newEventListBuffer = new StringBuilder(4000); var historyEventBatch = new TableBatchOperation(); EventType?orchestratorEventType = null; string eTagValue = this.GetETagValue(instanceId); DynamicTableEntity orchestrationInstanceUpdate = new DynamicTableEntity(instanceId, "") { Properties = { ["CustomStatus"] = new EntityProperty(runtimeState.Status), ["ExecutionId"] = new EntityProperty(executionId), ["LastUpdatedTime"] = new EntityProperty(newEvents.Last().Timestamp), } }; for (int i = 0; i < newEvents.Count; i++) { HistoryEvent historyEvent = newEvents[i]; DynamicTableEntity entity = this.tableEntityConverter.ConvertToTableEntity(historyEvent); await this.CompressLargeMessageAsync(entity); newEventListBuffer.Append(historyEvent.EventType.ToString()).Append(','); // The row key is the sequence number, which represents the chronological ordinal of the event. long sequenceNumber = i + (allEvents.Count - newEvents.Count); entity.RowKey = sequenceNumber.ToString("X16"); entity.PartitionKey = instanceId; entity.Properties["ExecutionId"] = new EntityProperty(executionId); // Replacement can happen if the orchestration episode gets replayed due to a commit failure in one of the steps below. historyEventBatch.InsertOrReplace(entity); // Table storage only supports inserts of up to 100 entities at a time. if (historyEventBatch.Count == 99) { eTagValue = await this.UploadHistoryBatch(instanceId, executionId, historyEventBatch, newEventListBuffer, newEvents.Count, eTagValue); // Reset local state for the next batch newEventListBuffer.Clear(); historyEventBatch.Clear(); } // Monitor for orchestration instance events switch (historyEvent.EventType) { case EventType.ExecutionStarted: orchestratorEventType = historyEvent.EventType; ExecutionStartedEvent executionStartedEvent = (ExecutionStartedEvent)historyEvent; orchestrationInstanceUpdate.Properties["Name"] = new EntityProperty(executionStartedEvent.Name); orchestrationInstanceUpdate.Properties["Version"] = new EntityProperty(executionStartedEvent.Version); orchestrationInstanceUpdate.Properties["CreatedTime"] = new EntityProperty(executionStartedEvent.Timestamp); orchestrationInstanceUpdate.Properties["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.Running.ToString()); this.SetTablePropertyForMessage(entity, orchestrationInstanceUpdate, InputProperty, InputProperty, executionStartedEvent.Input); break; case EventType.ExecutionCompleted: orchestratorEventType = historyEvent.EventType; ExecutionCompletedEvent executionCompleted = (ExecutionCompletedEvent)historyEvent; this.SetTablePropertyForMessage(entity, orchestrationInstanceUpdate, ResultProperty, OutputProperty, executionCompleted.Result); orchestrationInstanceUpdate.Properties["RuntimeStatus"] = new EntityProperty(executionCompleted.OrchestrationStatus.ToString()); break; case EventType.ExecutionTerminated: orchestratorEventType = historyEvent.EventType; ExecutionTerminatedEvent executionTerminatedEvent = (ExecutionTerminatedEvent)historyEvent; this.SetTablePropertyForMessage(entity, orchestrationInstanceUpdate, InputProperty, OutputProperty, executionTerminatedEvent.Input); orchestrationInstanceUpdate.Properties["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.Terminated.ToString()); break; case EventType.ContinueAsNew: orchestratorEventType = historyEvent.EventType; ExecutionCompletedEvent executionCompletedEvent = (ExecutionCompletedEvent)historyEvent; this.SetTablePropertyForMessage(entity, orchestrationInstanceUpdate, ResultProperty, OutputProperty, executionCompletedEvent.Result); orchestrationInstanceUpdate.Properties["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.ContinuedAsNew.ToString()); break; } } // First persistence step is to commit history to the history table. Messages must come after. if (historyEventBatch.Count > 0) { eTagValue = await this.UploadHistoryBatch(instanceId, executionId, historyEventBatch, newEventListBuffer, newEvents.Count, eTagValue); } if (orchestratorEventType == EventType.ExecutionCompleted || orchestratorEventType == EventType.ExecutionFailed || orchestratorEventType == EventType.ExecutionTerminated) { this.eTagValues.TryRemove(instanceId, out _); } else { this.eTagValues[instanceId] = eTagValue; } Stopwatch orchestrationInstanceUpdateStopwatch = Stopwatch.StartNew(); await this.instancesTable.ExecuteAsync(TableOperation.InsertOrMerge(orchestrationInstanceUpdate)); this.stats.StorageRequests.Increment(); this.stats.TableEntitiesWritten.Increment(); AnalyticsEventSource.Log.InstanceStatusUpdate( this.storageAccountName, this.taskHubName, instanceId, executionId, orchestratorEventType?.ToString() ?? string.Empty, orchestrationInstanceUpdateStopwatch.ElapsedMilliseconds); }
/// <summary> /// Inserts a set of new data entries into the table. /// Fails if the data does already exists. /// </summary> /// <param name="collection">Data entries to be inserted into the table.</param> /// <returns>Completion promise for this storage operation.</returns> public async Task BulkInsertTableEntries(IReadOnlyCollection <T> collection) { const string operation = "BulkInsertTableEntries"; if (collection == null) { throw new ArgumentNullException("collection"); } if (collection.Count > AzureTableDefaultPolicies.MAX_BULK_UPDATE_ROWS) { throw new ArgumentOutOfRangeException("collection", collection.Count, "Too many rows for bulk update - max " + AzureTableDefaultPolicies.MAX_BULK_UPDATE_ROWS); } if (collection.Count == 0) { return; } var startTime = DateTime.UtcNow; if (Logger.IsVerbose2) { Logger.Verbose2("Bulk inserting {0} entries to {1} table", collection.Count, TableName); } try { // WAS: // svc.AttachTo(TableName, entry); // svc.UpdateObject(entry); // SaveChangesOptions.None | SaveChangesOptions.Batch, // SaveChangesOptions.None == Insert-or-merge operation, SaveChangesOptions.Batch == Batch transaction // http://msdn.microsoft.com/en-us/library/hh452241.aspx var entityBatch = new TableBatchOperation(); foreach (T entry in collection) { entityBatch.Insert(entry); } try { // http://msdn.microsoft.com/en-us/library/hh452241.aspx await Task <IList <TableResult> > .Factory.FromAsync( tableReference.BeginExecuteBatch, tableReference.EndExecuteBatch, entityBatch, null); } catch (Exception exc) { Logger.Warn(ErrorCode.AzureTable_37, $"Intermediate error bulk inserting {collection.Count} entries in the table {TableName}", exc); } } finally { CheckAlertSlowAccess(startTime, operation); } }
internal static Tuple <HttpWebRequest, Stream> BuildRequestForTableBatchOperation(Uri uri, UriQueryBuilder builder, IBufferManager bufferManager, int?timeout, string tableName, TableBatchOperation batch, bool useVersionHeader, OperationContext ctx, TableRequestOptions options, string accountName) { HttpWebRequest msg = BuildRequestCore(NavigationHelper.AppendPathToSingleUri(uri, "$batch"), builder, "POST", timeout, useVersionHeader, ctx); TablePayloadFormat payloadFormat = options.PayloadFormat.Value; Logger.LogInformational(ctx, SR.PayloadFormat, payloadFormat); // create the writer, indent for readability of the examples. ODataMessageWriterSettings writerSettings = new ODataMessageWriterSettings() { CheckCharacters = false, // sets this flag on the XmlWriter for ATOM Version = TableConstants.ODataProtocolVersion // set the Odata version to use when writing the entry }; HttpWebRequestAdapterMessage adapterMsg = new HttpWebRequestAdapterMessage(msg, bufferManager); // Start Batch ODataMessageWriter odataWriter = new ODataMessageWriter(adapterMsg, writerSettings); ODataBatchWriter batchWriter = odataWriter.CreateODataBatchWriter(); batchWriter.WriteStartBatch(); bool isQuery = batch.Count == 1 && batch[0].OperationType == TableOperationType.Retrieve; // Query operations should not be inside changeset in payload if (!isQuery) { // Start Operation batchWriter.WriteStartChangeset(); batchWriter.Flush(); } foreach (TableOperation operation in batch) { string httpMethod = operation.HttpMethod; if (operation.OperationType == TableOperationType.Merge || operation.OperationType == TableOperationType.InsertOrMerge) { options.AssertNoEncryptionPolicyOrStrictMode(); httpMethod = "MERGE"; } ODataBatchOperationRequestMessage mimePartMsg = batchWriter.CreateOperationRequestMessage(httpMethod, operation.GenerateRequestURI(uri, tableName)); SetAcceptAndContentTypeForODataBatchMessage(mimePartMsg, payloadFormat); // etag if (operation.OperationType == TableOperationType.Delete || operation.OperationType == TableOperationType.Replace || operation.OperationType == TableOperationType.Merge) { mimePartMsg.SetHeader("If-Match", operation.Entity.ETag); } // Prefer header if (operation.OperationType == TableOperationType.Insert) { mimePartMsg.SetHeader("Prefer", operation.EchoContent ? "return-content" : "return-no-content"); } if (operation.OperationType != TableOperationType.Delete && operation.OperationType != TableOperationType.Retrieve) { using (ODataMessageWriter batchEntryWriter = new ODataMessageWriter(mimePartMsg, writerSettings, new TableStorageModel(accountName))) { // Write entity ODataWriter entryWriter = batchEntryWriter.CreateODataEntryWriter(); WriteOdataEntity(operation.Entity, operation.OperationType, ctx, entryWriter, options); } } } if (!isQuery) { // End Operation batchWriter.WriteEndChangeset(); } // End Batch batchWriter.WriteEndBatch(); batchWriter.Flush(); return(new Tuple <HttpWebRequest, Stream>(adapterMsg.GetPopulatedMessage(), adapterMsg.GetStream())); }
internal static IList <TableResult> TableBatchOperationPostProcess(IList <TableResult> result, TableBatchOperation batch, RESTCommand <IList <TableResult> > cmd, HttpWebResponse resp, OperationContext ctx, TableRequestOptions options, string accountName) { ODataMessageReaderSettings readerSettings = new ODataMessageReaderSettings(); readerSettings.MessageQuotas = new ODataMessageQuotas() { MaxPartsPerBatch = TableConstants.TableServiceMaxResults, MaxReceivedMessageSize = TableConstants.TableServiceMaxPayload }; using (ODataMessageReader responseReader = new ODataMessageReader(new HttpResponseAdapterMessage(resp, cmd.ResponseStream), readerSettings)) { // create a reader ODataBatchReader reader = responseReader.CreateODataBatchReader(); // Initial => changesetstart if (reader.State == ODataBatchReaderState.Initial) { reader.Read(); } if (reader.State == ODataBatchReaderState.ChangesetStart) { // ChangeSetStart => Operation reader.Read(); } int index = 0; bool failError = false; bool failUnexpected = false; while (reader.State == ODataBatchReaderState.Operation) { TableOperation currentOperation = batch[index]; TableResult currentResult = new TableResult() { Result = currentOperation.Entity }; result.Add(currentResult); ODataBatchOperationResponseMessage mimePartResponseMessage = reader.CreateOperationResponseMessage(); string contentType = mimePartResponseMessage.GetHeader(Constants.ContentTypeElement); currentResult.HttpStatusCode = mimePartResponseMessage.StatusCode; // Validate Status Code. if (currentOperation.OperationType == TableOperationType.Insert) { failError = mimePartResponseMessage.StatusCode == (int)HttpStatusCode.Conflict; if (currentOperation.EchoContent) { failUnexpected = mimePartResponseMessage.StatusCode != (int)HttpStatusCode.Created; } else { failUnexpected = mimePartResponseMessage.StatusCode != (int)HttpStatusCode.NoContent; } } else if (currentOperation.OperationType == TableOperationType.Retrieve) { if (mimePartResponseMessage.StatusCode == (int)HttpStatusCode.NotFound) { index++; // Operation => next reader.Read(); continue; } failUnexpected = mimePartResponseMessage.StatusCode != (int)HttpStatusCode.OK; } else { failError = mimePartResponseMessage.StatusCode == (int)HttpStatusCode.NotFound; failUnexpected = mimePartResponseMessage.StatusCode != (int)HttpStatusCode.NoContent; } if (failError) { // If the parse error is null, then don't get the extended error information and the StorageException will contain SR.ExtendedErrorUnavailable message. if (cmd.ParseError != null) { cmd.CurrentResult.ExtendedErrorInformation = cmd.ParseError(mimePartResponseMessage.GetStream(), resp, contentType); } cmd.CurrentResult.HttpStatusCode = mimePartResponseMessage.StatusCode; if (!string.IsNullOrEmpty(cmd.CurrentResult.ExtendedErrorInformation.ErrorMessage)) { string msg = cmd.CurrentResult.ExtendedErrorInformation.ErrorMessage; cmd.CurrentResult.HttpStatusMessage = msg.Substring(0, msg.IndexOf("\n", StringComparison.Ordinal)); } else { cmd.CurrentResult.HttpStatusMessage = mimePartResponseMessage.StatusCode.ToString(CultureInfo.InvariantCulture); } throw new StorageException( cmd.CurrentResult, cmd.CurrentResult.ExtendedErrorInformation != null ? cmd.CurrentResult.ExtendedErrorInformation.ErrorMessage : SR.ExtendedErrorUnavailable, null) { IsRetryable = false }; } if (failUnexpected) { // If the parse error is null, then don't get the extended error information and the StorageException will contain SR.ExtendedErrorUnavailable message. if (cmd.ParseError != null) { cmd.CurrentResult.ExtendedErrorInformation = cmd.ParseError(mimePartResponseMessage.GetStream(), resp, contentType); } cmd.CurrentResult.HttpStatusCode = mimePartResponseMessage.StatusCode; if (!string.IsNullOrEmpty(cmd.CurrentResult.ExtendedErrorInformation.ErrorMessage)) { string msg = cmd.CurrentResult.ExtendedErrorInformation.ErrorMessage; cmd.CurrentResult.HttpStatusMessage = msg.Substring(0, msg.IndexOf("\n", StringComparison.Ordinal)); } else { cmd.CurrentResult.HttpStatusMessage = mimePartResponseMessage.StatusCode.ToString(CultureInfo.InvariantCulture); } string indexString = Convert.ToString(index, CultureInfo.InvariantCulture); // Attempt to extract index of failing entity from extended error info if (cmd.CurrentResult.ExtendedErrorInformation != null && !string.IsNullOrEmpty(cmd.CurrentResult.ExtendedErrorInformation.ErrorMessage)) { string tempIndex = TableRequest.ExtractEntityIndexFromExtendedErrorInformation(cmd.CurrentResult); if (!string.IsNullOrEmpty(tempIndex)) { indexString = tempIndex; } } throw new StorageException(cmd.CurrentResult, string.Format(CultureInfo.CurrentCulture, SR.BatchErrorInOperation, indexString), null) { IsRetryable = true }; } // Update etag if (!string.IsNullOrEmpty(mimePartResponseMessage.GetHeader(Constants.HeaderConstants.EtagHeader))) { currentResult.Etag = mimePartResponseMessage.GetHeader(Constants.HeaderConstants.EtagHeader); if (currentOperation.Entity != null) { currentOperation.Entity.ETag = currentResult.Etag; } } // Parse Entity if needed if (currentOperation.OperationType == TableOperationType.Retrieve || (currentOperation.OperationType == TableOperationType.Insert && currentOperation.EchoContent)) { if (mimePartResponseMessage.GetHeader(Constants.ContentTypeElement).Contains(Constants.JsonNoMetadataAcceptHeaderValue)) { ReadEntityUsingJsonParser(currentResult, currentOperation, mimePartResponseMessage.GetStream(), ctx, options); } else { ReadOdataEntity(currentResult, currentOperation, mimePartResponseMessage, ctx, readerSettings, accountName, options); } } else if (currentOperation.OperationType == TableOperationType.Insert) { currentOperation.Entity.Timestamp = ParseETagForTimestamp(currentResult.Etag); } index++; // Operation => reader.Read(); } } return(result); }
public void Execute(TableBatchOperation batchOperation, TableEntity entity) { batchOperation.InsertOrReplace(entity); }
public override Task <IList <TableResult> > ExecuteBatchAsync(TableBatchOperation batch, TableRequestOptions requestOptions = null, OperationContext operationContext = null) { //Trace.TraceInformation("{0} calling {1}.ExecuteBatchAsync({2})", callerMachineId, debugName, BetterComparer.ToString(batch)); return(tableCallEventProxy.ExecuteBatchAsync(batch, requestOptions, operationContext)); }
async void Initialize() { // TODO: Figure out how to verify the transition from // "old table only" to "in migration" to "new table only". // (I don't think this is the biggest risk, but I'm still // interested in verifying it.) // I assume once we do that, the possibility of insertions while // we're in "old table only" will model the ability to start // from a nonempty table. configService = new InMemoryConfigurationService <MTableConfiguration>( MasterMigratingTable.INITIAL_CONFIGURATION); oldTable = new InMemoryTable(); newTable = new InMemoryTable(); referenceTable = new InMemoryTableWithHistory(); #if false // Second partition from the example in: // https://microsoft.sharepoint.com/teams/toolsforeng/Shared%20Documents/ContentRepository/LiveMigration/Migration_slides.pptx MTableEntity eMeta = new MTableEntity { PartitionKey = MigrationModel.SINGLE_PARTITION_KEY, RowKey = MigratingTable.ROW_KEY_PARTITION_META, partitionState = MTablePartitionState.SWITCHED, }; MTableEntity e0 = TestUtils.CreateTestMTableEntity("0", "orange"); MTableEntity e1old = TestUtils.CreateTestMTableEntity("1", "red"); MTableEntity e2new = TestUtils.CreateTestMTableEntity("2", "green"); MTableEntity e3old = TestUtils.CreateTestMTableEntity("3", "blue"); MTableEntity e3new = TestUtils.CreateTestMTableEntity("3", "azure"); MTableEntity e4old = TestUtils.CreateTestMTableEntity("4", "yellow"); MTableEntity e4new = TestUtils.CreateTestMTableEntity("4", null, true); var oldBatch = new TableBatchOperation(); oldBatch.InsertOrReplace(eMeta); oldBatch.InsertOrReplace(e0); oldBatch.InsertOrReplace(e1old); oldBatch.InsertOrReplace(e3old); oldBatch.InsertOrReplace(e4old); IList <TableResult> oldTableResult = await oldTable.ExecuteBatchAsync(oldBatch); await ExecuteExportedMirrorBatchAsync(oldBatch, oldTableResult); var newBatch = new TableBatchOperation(); newBatch.InsertOrReplace(e0); newBatch.InsertOrReplace(e2new); newBatch.InsertOrReplace(e3new); newBatch.InsertOrReplace(e4new); IList <TableResult> newTableResult = await newTable.ExecuteBatchAsync(newBatch); // Allow rows to overwrite rather than composing the virtual ETags manually. // InsertOrReplace doesn't use the ETag, so we don't care that the ETag was mutated by the original batch. await ExecuteExportedMirrorBatchAsync(newBatch, newTableResult); #endif // Start with the old table now. var batch = new TableBatchOperation(); batch.InsertOrReplace(TestUtils.CreateTestEntity("0", "orange")); batch.InsertOrReplace(TestUtils.CreateTestEntity("1", "red")); batch.InsertOrReplace(TestUtils.CreateTestEntity("3", "blue")); batch.InsertOrReplace(TestUtils.CreateTestEntity("4", "yellow")); IList <TableResult> oldTableResult = await oldTable.ExecuteBatchAsync(batch); // InsertOrReplace doesn't use the ETag, so we don't care that the ETag was mutated by the original batch. await referenceTable.ExecuteMirrorBatchAsync(batch, oldTableResult); //CreateMonitor(typeof(RunningServiceMachinesMonitor)); for (int i = 0; i < MigrationModel.NUM_SERVICE_MACHINES; i++) { InitializeAppMachine(CreateMachine(typeof(ServiceMachine))); } InitializeAppMachine(CreateMachine(typeof(MigratorMachine))); Send(Id, new TablesMachineInitializedEvent()); }
public override Task <IList <TableResult> > ExecuteBatchAsync(TableBatchOperation batch, TableRequestOptions requestOptions = null, OperationContext operationContext = null) { return(ExecuteMirrorBatchAsync(batch, null, requestOptions, operationContext)); }
public abstract Task <IList <TableResult> > ExecuteMirrorBatchAsync( TableBatchOperation originalBatch, IList <TableResult> originalResponse, TableRequestOptions requestOptions = null, OperationContext operationContext = null);
async Task DoRandomAtomicCalls() { for (int callNum = 0; callNum < MigrationModel.NUM_CALLS_PER_MACHINE; callNum++) { TableCall originalCall; MirrorTableCall referenceCall; SortedDictionary <PrimaryKey, DynamicTableEntity> dump = await peekProxy.DumpReferenceTableAsync(); if (PSharpRuntime.Nondeterministic()) { // Query // XXX: Test the filtering? var query = new TableQuery <DynamicTableEntity>(); query.FilterString = TableQuery.GenerateFilterCondition( TableConstants.PartitionKey, QueryComparisons.Equal, MigrationModel.SINGLE_PARTITION_KEY); // async/await pair needed to upcast the return value to object. originalCall = async table => await table.ExecuteQueryAtomicAsync(query); referenceCall = async referenceTable => await referenceTable.ExecuteQueryAtomicAsync(query); Console.WriteLine("{0} starting query", machineId); } else { // Batch write int batchSize = PSharpRuntime.Nondeterministic() ? 2 : 1; var batch = new TableBatchOperation(); var rowKeyChoices = new List <string> { "0", "1", "2", "3", "4", "5" }; for (int opNum = 0; opNum < batchSize; opNum++) { int opTypeNum = PSharpNondeterminism.Choice(7); int rowKeyI = PSharpNondeterminism.Choice(rowKeyChoices.Count); string rowKey = rowKeyChoices[rowKeyI]; rowKeyChoices.RemoveAt(rowKeyI); // Avoid duplicate in same batch var primaryKey = new PrimaryKey(MigrationModel.SINGLE_PARTITION_KEY, rowKey); string eTag = null; if (opTypeNum >= 1 && opTypeNum <= 3) { DynamicTableEntity existingEntity; int etagTypeNum = PSharpNondeterminism.Choice( dump.TryGetValue(primaryKey, out existingEntity) ? 3 : 2); switch (etagTypeNum) { case 0: eTag = ChainTable2Constants.ETAG_ANY; break; case 1: eTag = "wrong"; break; case 2: eTag = existingEntity.ETag; break; } } DynamicTableEntity entity = new DynamicTableEntity { PartitionKey = MigrationModel.SINGLE_PARTITION_KEY, RowKey = rowKey, ETag = eTag, Properties = new Dictionary <string, EntityProperty> { // Give us something to see on merge. Might help with tracing too! { string.Format("{0}_c{1}_o{2}", machineId.ToString(), callNum, opNum), new EntityProperty(true) } } }; switch (opTypeNum) { case 0: batch.Insert(entity); break; case 1: batch.Replace(entity); break; case 2: batch.Merge(entity); break; case 3: batch.Delete(entity); break; case 4: batch.InsertOrReplace(entity); break; case 5: batch.InsertOrMerge(entity); break; case 6: entity.ETag = ChainTable2Constants.ETAG_DELETE_IF_EXISTS; batch.Delete(entity); break; } } TableBatchOperation batchCopy = ChainTableUtils.CopyBatch <DynamicTableEntity>(batch); originalCall = async table => await table.ExecuteBatchAsync(batch); referenceCall = async referenceTable => await referenceTable.ExecuteMirrorBatchAsync(batchCopy, successfulBatchResult); Console.WriteLine("{0} starting batch {1}", machineId, batch); } await RunCallAsync(originalCall, referenceCall); Console.WriteLine("{0} table call verified"); } }
public IList <TableResult> ExecuteBatch(TableBatchOperation batch, TableRequestOptions requestOptions = null, OperationContext operationContext = null) { return(batch.Select(operation => Execute(operation)).ToList()); }
public ICancellableAsyncResult BeginExecuteBatch(TableBatchOperation batch, AsyncCallback callback, object state) { return(BeginExecuteBatch(batch, null, null, callback, state)); }