public List <CustomerEntity> All() { TableQuery <CustomerEntity> tableQuery = new TableQuery <CustomerEntity>(); TableContinuationToken continuationToken = null; var fullList = new List <CustomerEntity>(); do { TableQuerySegment <CustomerEntity> result = _azureTable.ExecuteQuerySegmented(tableQuery, continuationToken); continuationToken = result.ContinuationToken; var customerList = result as IList <CustomerEntity> ?? result.ToList(); fullList.AddRange(customerList); } while (continuationToken != null); return(fullList); }
public IEnumerable <T> GetAll <T>() where T : TableEntity, new() { TableQuerySegment <T> querySegment = null; while (querySegment == null || querySegment.ContinuationToken != null) { querySegment = m_table.ExecuteQuerySegmented( new TableQuery <T>(), querySegment != null ? querySegment.ContinuationToken : null ); foreach (var entity in querySegment.Results) { yield return(entity); } } }
public ActionResult Index() { TableQuery <ProductsFromTable> query = new TableQuery <ProductsFromTable>(); List <ProductsFromTable> products = new List <ProductsFromTable>(); TableContinuationToken token = null; do { TableQuerySegment <ProductsFromTable> resultSegment = table.ExecuteQuerySegmented(query, token); token = resultSegment.ContinuationToken; foreach (ProductsFromTable product in resultSegment.Results) { products.Add(product); } } while (token != null); return(View(products)); }
/// <summary> /// Get all the records in the table /// </summary> /// <returns>All records</returns> public IEnumerable <T> GetAllRecords() { var query = new TableQuery <T>(); var token = new TableContinuationToken(); var segment = CloudTable.ExecuteQuerySegmented(query, token); while (token != null) { foreach (var result in segment) { yield return(result); } token = segment.ContinuationToken; segment = CloudTable.ExecuteQuerySegmented(query, token); } }
static void QueryAllRecords() { TableQuery <employee> tq = new TableQuery <employee>().Where( TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "Hyderabad"), TableOperators.And, TableQuery.GenerateFilterCondition("firstname", QueryComparisons.Equal, "rrr") ) ); TableQuerySegment <employee> tqs = ct.ExecuteQuerySegmented <employee>(tq, new TableContinuationToken()); foreach (var data in tqs) { Console.WriteLine(data.firstname + " " + data.lastname + " " + data.PartitionKey); } }
static List <DynamicTableEntity> RetrieveAll(CloudTable table, IQueryable <DynamicTableEntity> query) { var entities = new List <DynamicTableEntity>(); TableContinuationToken token = null; do { var page = query.Take(512); var segment = table.ExecuteQuerySegmented((TableQuery <DynamicTableEntity>)page, token); token = segment.ContinuationToken; entities.AddRange(segment.Results); }while (token != null); return(entities); }
private IEnumerable <DynamicTableEntity> ExecuteBalanceQuery(CloudTable table, TableQuery tableQuery, IEnumerable <int> pages) { pages = pages ?? new int[0]; var pagesEnumerator = pages.GetEnumerator(); TableContinuationToken continuation = null; do { tableQuery.TakeCount = pagesEnumerator.MoveNext() ? (int?)pagesEnumerator.Current : null; var segment = table.ExecuteQuerySegmented(tableQuery, continuation); continuation = segment.ContinuationToken; foreach (var entity in segment) { yield return(entity); } } while(continuation != null); }
/// <summary> /// Performs the <paramref name="action"/> on a <paramref name="table"/> filtered by <paramref name="filter"/>. /// </summary> /// <typeparam name="TEntity">The type of the entities in the <paramref name="table"/>.</typeparam> /// <param name="table">The cloud table.</param> /// <param name="action">The action to perform on the entities.</param> /// <param name="filter">A filter for selecting the entities.</param> public static void ProcessEntities <TEntity>(this CloudTable table, Action <IEnumerable <TEntity> > action, Expression <Func <TEntity, bool> > filter) where TEntity : ITableEntity, new() { TableQuerySegment <TEntity> segment = null; while (segment == null || segment.ContinuationToken != null) { if (filter == null) { segment = table.ExecuteQuerySegmented(new TableQuery <TEntity>().Take(100), segment?.ContinuationToken); } else { var query = table.CreateQuery <TEntity>().Where(filter).Take(100).AsTableQuery(); segment = query.ExecuteSegmented(segment?.ContinuationToken); } action(segment.Results); } }
private void LoadFromPersistentStorage(CloudTable table) { TableContinuationToken?token = null; var entities = new List <DynamicTableEntity>(); do { var queryResult = table.ExecuteQuerySegmented(new TableQuery(), token); entities.AddRange(queryResult.Results); token = queryResult.ContinuationToken; } while (token != null); foreach (var item in entities) { var vehicleJourneyAssignment = item.ToVehicleJourneyAssignment(); vehicleJourneyAssignmentCache.Put(vehicleJourneyAssignment); } }
//Returns a list of all Items in the items table public static List <Item> GetItems() { CloudTable table = GetCloudTable(); TableQuery <Item> query = new TableQuery <Item>(); List <Item> itemsLst = new List <Item>(); TableContinuationToken token = null; do { TableQuerySegment <Item> resultSegment = table.ExecuteQuerySegmented(query, token); token = resultSegment.ContinuationToken; foreach (Item item in resultSegment.Results) { itemsLst.Add(item); } } while (token != null); return(itemsLst); }
public Tuple <List <T>, TableContinuationToken> GetAllEntitiesByBatchs <T>(int maxNumOfElements, TableContinuationToken continuationToken) where T : MyStateTableEntity, new() { // Create the CloudTable object that represents the "people" table. CloudTable table = GetTableName <T>(); if (table == null) { return(null); } var tableQuery = new TableQuery <T>(); tableQuery.Take(maxNumOfElements); var query = table.ExecuteQuerySegmented( tableQuery, continuationToken); return(new Tuple <List <T>, TableContinuationToken>(query.Results, query.ContinuationToken)); }
/// <summary> /// Changes user validation and unsubscribe codes. /// </summary> /// <param name="subscriberTable">The subscriber table.</param> public static void RefreshUserValidationStrings(CloudTable subscriberTable) { try { var query = (from record in subscriberTable.CreateQuery <DynamicTableEntity>() where record.PartitionKey == ApplicationConstants.SubscriberListKey && record.Properties["IsVerified"].BooleanValue == true select record).Take(100).AsTableQuery(); TableContinuationToken token = null; do { var segment = subscriberTable.ExecuteQuerySegmented(query, token, TableRequestOptions); var batchUpdate = new TableBatchOperation(); if (null == segment || !segment.Any()) { Console.Out.WriteLine("No users found. Aborting current call."); } else { //// Log users we are going to update. foreach (var user in segment) { Console.Out.WriteLine("Updating {0}", user.Properties["Email"].StringValue); user.Properties["VerificationString"].StringValue = Guid.NewGuid().ToString(); user.Properties["UnsubscribeString"].StringValue = Guid.NewGuid().ToString(); batchUpdate.Add(TableOperation.Replace(user)); } Console.Out.WriteLine("Going to update user secrets"); subscriberTable.ExecuteBatch(batchUpdate, TableRequestOptions); token = segment.ContinuationToken; } }while (token != null); Console.Out.WriteLine("Secrets of all users updated."); } catch (Exception exception) { Console.Error.WriteLine("Error at Time:{0} Exception:{1}", DateTime.UtcNow, exception); throw; } }
/// <summary> /// Get the number of the records in the table /// </summary> /// <returns>The record count</returns> public int GetRecordCount() { TableContinuationToken continuationToken = null; var query = new TableQuery().Select(new List <string> { "PartitionKey" }); var recordCount = 0; do { var items = CloudTable.ExecuteQuerySegmented(query, continuationToken); continuationToken = items.ContinuationToken; recordCount += items.Count(); } while (continuationToken != null); return(recordCount); }
//Returns the list of all departments in the departments table public static List <Department> GetDepartments() { CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); CloudTable table = tableClient.GetTableReference(DEP_TABLE_NAME); TableQuery <Department> query = new TableQuery <Department>(); List <Department> departmentsLst = new List <Department>(); TableContinuationToken token = null; do { TableQuerySegment <Department> resultSegment = table.ExecuteQuerySegmented(query, token); token = resultSegment.ContinuationToken; foreach (Department dept in resultSegment.Results) { departmentsLst.Add(dept); } } while (token != null); return(departmentsLst); }
private IEnumerable <StorageEntity> GetAllEntities(CloudTable table) { List <StorageEntity> result = new List <StorageEntity>(); // thanks to https://stackoverflow.com/a/48227035/2023653 var criteria = TableQuery.GenerateFilterCondition(nameof(StorageEntity.PartitionKey), QueryComparisons.Equal, PartitionKey); var query = new TableQuery <StorageEntity>().Where(criteria); TableContinuationToken continuationToken = null; do { var results = table.ExecuteQuerySegmented(query, continuationToken); foreach (var item in results) { result.Add(item); } continuationToken = results.ContinuationToken; }while (continuationToken != null); return(result); }
public static async Task <IEnumerable <TEntity> > ExecuteQuery <TEntity>( this CloudTable table, TableQuery <TEntity> query, CancellationToken cancellationToken) where TEntity : ITableEntity, new() { var entities = new List <TEntity>(); TableContinuationToken continuation = null; do { TableQuerySegment <TEntity> segment = await table .ExecuteQuerySegmented(query, continuation, cancellationToken) .ConfigureAwait(false); entities.AddRange(segment); continuation = segment.ContinuationToken; }while (continuation != null); return(entities); }
private void ProcessEntities(CloudTable table, Action <IEnumerable <DynamicTableEntity>, CloudTable> processor, Expression <Func <DynamicTableEntity, bool> > filter) { TableQuerySegment <DynamicTableEntity> segment = null; while (segment == null || segment.ContinuationToken != null) { if (filter == null) { segment = table.ExecuteQuerySegmented(new TableQuery().Take(100), segment == null ? null : segment.ContinuationToken); } else { var query = table.CreateQuery <DynamicTableEntity>().Where(filter).Take(100).AsTableQuery(); segment = query.ExecuteSegmented(segment == null ? null : segment.ContinuationToken); } processor(segment.Results, table); } }
public async Task FlushAllPendingEvents(CancellationToken cancellationToken) { string filter = PendingEvent.FullScanFilter; var query = new TableQuery <PendingEvent> { FilterString = filter }; TableContinuationToken continuation = null; do { TableQuerySegment <PendingEvent> segment = await _eventTable .ExecuteQuerySegmented(query, continuation, cancellationToken) .ConfigureAwait(false); foreach (string partition in segment.Select(e => e.PartitionKey).Distinct()) { await Flush(partition, cancellationToken).ConfigureAwait(false); } continuation = segment.ContinuationToken; }while (continuation != null); }
//<summary> //Select a random RowKey from a given PartitionKey //</summary> public string GetRandomRowKey(CloudTable table, string PartitionKey) { try { TableContinuationToken continuationToken = null; var EntityRecords = new List <TableRecord>(); do { var rangeQueryResult = table .ExecuteQuerySegmented(new TableQuery <TableRecord>() .Where(TableQuery .GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, PartitionKey)), continuationToken); EntityRecords.AddRange(rangeQueryResult.Results); continuationToken = rangeQueryResult.ContinuationToken; } while (continuationToken != null); Dictionary <int, string> RowKeys = new Dictionary <int, string>(); int number = 1; foreach (var Entity in EntityRecords) { RowKeys.Add(number, Entity.RowKey); number++; } int subSize = RowKeys.Count; Random rand = new Random(); const int MinValue = 1; int rand_number = rand.Next(MinValue, subSize + 1); string RandomRowKey = RowKeys[rand_number]; return(RandomRowKey); } catch (Exception ex) { Console.WriteLine(ex.Message); Console.WriteLine(ex.Source); Console.WriteLine(ex.StackTrace); return(null); } }
public List <listcabackupconfig> GetBackups() { var continuationToken = new TableContinuationToken(); List <listcabackupconfig> config = new List <listcabackupconfig>(); do { var queryResult = table.ExecuteQuerySegmented(new TableQuery <listcabackupconfig>(), continuationToken); foreach (var backupconfig in queryResult.Results) { string policy = ""; if (backupconfig.PartitionKey == "CABackupDaily") { policy = JObject.Parse(backupconfig.CABackup)["displayName"].ToString(); } else if (backupconfig.PartitionKey == "CABackupChanges") { policy = JObject.Parse(backupconfig.CAOldValue)["displayName"].ToString(); } config.Add(new listcabackupconfig { PartitionKey = backupconfig.PartitionKey, RowKey = backupconfig.RowKey, BackupDate = backupconfig.BackupDate, Policy = policy, BackupRestoreing = backupconfig.BackupRestoreing }); } continuationToken = queryResult.ContinuationToken; } while (continuationToken != null); return(config); }
private List <AcrCallback> getLast(string streamId = TILOSHU_STREAMID, int limit = 1, int offset = 0) { CloudTable table = getTable(); TableQuery <AcrCallback> query = new TableQuery <AcrCallback>().Where(getPartitionFilter(streamId)); List <AcrCallback> playlist = new List <AcrCallback>(); TableContinuationToken token = null; do { TableQuerySegment <AcrCallback> resultSegment = table.ExecuteQuerySegmented(query, token); token = resultSegment.ContinuationToken; foreach (AcrCallback customer in resultSegment.Results) { playlist.Add(customer); } } while (token != null && playlist.Count < (limit + offset)); var result = playlist.Skip(offset).Take(limit); return(result.ToList()); }
private List <ZMTA_SALES> GetOrderById(string orderid) { List <ZMTA_SALES> orders = null; InitializeCloudClient(); // Retrieve cloud table by tablename. CloudTable table = cloudTableClient.GetTableReference("sapdatatable"); // Create table Query TableQuery <ZMTA_SALES> query = new TableQuery <ZMTA_SALES>(); string partitionFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, orderid); TableContinuationToken continuationToken = null; var page = table.ExecuteQuerySegmented(query.Where(partitionFilter), continuationToken); if (page.Results != null) { orders = new List <ZMTA_SALES>(); orders.AddRange(page.Results); } return(orders); }
public override List <Event> Get(Guid aggregateId) { TableQuery <TableEvent> query = new TableQuery <TableEvent>() .Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, aggregateId.ToString())); List <Event> allEvents = new List <Event>(); TableContinuationToken token = new TableContinuationToken(); do { TableQuerySegment <TableEvent> events = _cloudTable.ExecuteQuerySegmented(query, token); token = events.ContinuationToken; foreach (var @event in events.Results) { allEvents.Add(ToEvent(@event)); } } while (token != null); return(allEvents .OrderBy(e => e.Order) .ToList()); }
public IActionResult Index() { var connectionstring = "DefaultEndpointsProtocol=https;AccountName=storageaccountkursu9c74;AccountKey=0UKdJ8IwdyLWE30jU43KGqnRu0n73sHORUHfVAE3XgVLHnZwbcez0MKpSOHs0IBNH6jUV9uBgA4gUSzq7LOfGQ==;EndpointSuffix=core.windows.net"; CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(connectionstring); CloudTableClient cloudTableClient = cloudStorageAccount.CreateCloudTableClient(); CloudTable cloudTable = cloudTableClient.GetTableReference("customers"); cloudTable.CreateIfNotExists(); Customer customer = new Customer("Skovby", Guid.NewGuid().ToString()) { Email = "*****@*****.**", FirstName = "Rune", LastName = "Klan" }; TableOperation operation = TableOperation.Insert(customer); cloudTable.Execute(operation); List <Customer> customers = new List <Customer>(); var hasMore = true; while (hasMore) { TableContinuationToken token = null; var queryResult = cloudTable.ExecuteQuerySegmented(new TableQuery <Customer>(), token); customers.AddRange(queryResult.Results); token = queryResult.ContinuationToken; hasMore = token != null; } return(View(customers)); }
/// <summary> /// Private extension method that performs an asynchronous query to an Azure table and provides progress callbacks while executing an Azure query asynchronously. /// </summary> /// <typeparam name="T">The type of table entity being returned.</typeparam> /// <param name="table">The <see cref="CloudTable"/> to query.</param> /// <param name="query">The <see cref="TableQuery"/> to send to Azure.</param> /// <param name="ct">An optional <see cref="CancellationToken"/> that can be used to cancel the query.</param> /// <param name="onProgress">An optional callback method to call as query results are coming in.</param> /// <param name="onDone">An optional callback method to call when the query has completed.</param> /// <returns>A <see cref="Task"/> that when complete contains the list of query results.</returns> private static Task <IList <T> > ExecuteQueryAsync <T>(this CloudTable table, TableQuery <T> query, CancellationToken ct = default(CancellationToken), Action <IList <T> > onProgress = null, Action <IList <T>, bool> onDone = null) where T : ITableEntity, new() { return(Task.Run(() => { var runningQuery = new TableQuery <T>() { FilterString = query.FilterString, SelectColumns = query.SelectColumns }; var items = new List <T>(); TableContinuationToken token = null; do { runningQuery.TakeCount = query.TakeCount - items.Count; Debug.WriteLine("Executing Query [" + query.FilterString + "] with Continuation Token [" + token?.NextTableName + token?.NextPartitionKey + token?.NextRowKey + "]"); TableQuerySegment <T> seg = table.ExecuteQuerySegmented <T>(runningQuery, token); token = seg.ContinuationToken; Debug.WriteLine("Query Returned " + seg.Results.Count + " Results, with new Continuation Token [" + seg.ContinuationToken?.NextTableName + seg.ContinuationToken?.NextPartitionKey + seg.ContinuationToken?.NextRowKey + "]"); items.AddRange(seg); if (onProgress != null) { onProgress(seg.Results); } } while (token != null && !ct.IsCancellationRequested && (query.TakeCount == null || items.Count < query.TakeCount.Value)); if (onDone != null) { onDone(items, ct.IsCancellationRequested); } return (IList <T>)items; })); }
/// <summary> /// Deletes a column from an Azure table. /// The table for the column is same as the input table from the dataset /// The column to be deleted is specified using the following extended properties /// Extended Properties /// columnName - Name of the column to be deleted /// rowKeyPrefix - Rowkey prefix of the row from which the column will be deleted. This is optional and will identify the subset of rows to do this operation. /// columnName is mandatory. /// Extended Properties Example /// "columnName": "UseDefault", /// "rowKeyPrefix": "IdentityCredentialsObject:" /// Activity Operation /// The activity iterates through all the rows from the input table with the matching rowKeyPrefix, /// checks for the column, removes the column if found and runs a replace table operation to replace the contents of /// row/entity in the table. /// </summary> /// <param name="linkedServices">Linked services referenced by activity definition.</param> /// <param name="datasets">Datasets referenced by activity definition.</param> /// <param name="activity">Activity definition.</param> /// <param name="logger">Used to log messages during activity execution.</param> /// <returns>Activity state at the end of execution</returns> public IDictionary <string, string> Execute( IEnumerable <LinkedService> linkedServices, IEnumerable <Dataset> datasets, Activity activity, IActivityLogger logger) { DotNetActivity dotNetActivity = (DotNetActivity)activity.TypeProperties; IDictionary <string, string> extendedProperties = dotNetActivity.ExtendedProperties; logger.Write("Logging extended properties if any..."); foreach (KeyValuePair <string, string> entry in extendedProperties) { logger.Write("<key:{0}> <value:{1}>", entry.Key, entry.Value); } if (!extendedProperties.ContainsKey("columnName")) { throw new ArgumentException("Column name is required", "columnName"); } string columnName = extendedProperties["columnName"]; string rowKeyPrefix = string.Empty; if (extendedProperties.ContainsKey("rowKeyPrefix")) { rowKeyPrefix = extendedProperties["rowKeyPrefix"]; } AzureStorageLinkedService inputLinkedService; AzureTableDataset sourceTable; // For activities working on a single dataset, the first entry is the input dataset. // The activity.Inputs can have multiple datasets for building pipeline workflow dependencies. We can ignore the rest of the datasets Dataset inputDataset = datasets.Single(dataset => dataset.Name == activity.Inputs.First().Name); sourceTable = inputDataset.Properties.TypeProperties as AzureTableDataset; logger.Write("input table:{0}", sourceTable.TableName); inputLinkedService = linkedServices.First( ls => ls.Name == inputDataset.Properties.LinkedServiceName).Properties.TypeProperties as AzureStorageLinkedService; string inputConnectionString = inputLinkedService.ConnectionString; // create storage client for input. Pass the connection string. CloudStorageAccount inputStorageAccount = CloudStorageAccount.Parse(inputConnectionString); CloudTableClient inputTableClient = inputStorageAccount.CreateCloudTableClient(); CloudTable inputTable = inputTableClient.GetTableReference(sourceTable.TableName); long totalProcessedRecords = 0; long actualAffectedRecords = 0; TableContinuationToken tableContinuationToken = null; List <Task> tasks = new List <Task>(); do { var resultSegment = inputTable.ExecuteQuerySegmented(new TableQuery(), tableContinuationToken); tableContinuationToken = resultSegment.ContinuationToken; var partitionGroups = (from s in resultSegment.Results where string.IsNullOrWhiteSpace(rowKeyPrefix) ? true : s.RowKey.StartsWith(rowKeyPrefix) select s).GroupBy(a => a.PartitionKey); foreach (IGrouping <string, DynamicTableEntity> g in partitionGroups) { TableBatchOperation batch = new TableBatchOperation(); foreach (DynamicTableEntity e in g.AsEnumerable()) { // If the columnName exist in the properties, then Remove it if (e.Properties.ContainsKey(columnName)) { e.Properties.Remove(columnName); batch.Replace(e); logger.Write("<partition key:{0}>, <row key:{1}> added to batch", e.PartitionKey, e.RowKey); } } if (batch.Count > 0) { tasks.Add(inputTable.ExecuteBatchInChunkAsync(batch)); actualAffectedRecords += batch.Count; } } totalProcessedRecords += resultSegment.Results.Count; logger.Write("Processed records count: {0}", totalProcessedRecords); logger.Write("Affected records count: {0}", actualAffectedRecords); }while (tableContinuationToken != null); // The batch operations complete when Task.WaitAll completes Task.WaitAll(tasks.ToArray()); logger.Write("Deleted column from {0} records", actualAffectedRecords); return(new Dictionary <string, string>()); }
public void SearchForDuplicates(Action <Guid, IEnumerable <Guid> > onCollision) { var query = new TableQuery { SelectColumns = columns }; TableContinuationToken token = null; var buffers = new List <IdHashBuffer>(); var buffer = new IdHashBuffer(InitialBufferSize); buffers.Add(buffer); var collisionBytes = new byte[4096]; var ms = new MemoryStream(collisionBytes); var sw = new StreamWriter(ms); do { var executeQuerySegmented = table.ExecuteQuerySegmented(query, token); foreach (var dte in executeQuerySegmented.Results) { var id = Guid.Parse(dte.PartitionKey); EntityProperty property; if (dte.Properties.TryGetValue(indexPropertyName, out property)) { var hash = hashingTransformer(property.PropertyAsObject); if (buffer.TryWrite(id, hash) == false) { buffer = new IdHashBuffer(buffer.Size * 2); buffers.Add(buffer); if (buffer.TryWrite(id, hash) == false) { throw new OutOfMemoryException(); } } } } token = executeQuerySegmented.ContinuationToken; } while (token != null); foreach (var b in buffers) { b.Seal(); } for (var i = 0; i < buffers.Count; i++) { var b = buffers[i]; b.FindHashCollisions(buffers.Skip(i), (hash, ids) => { var collisions = ids.Select(id => new TableQuery { FilterString = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, id.ToString()), SelectColumns = columns }) .Select(q => table.ExecuteQuery(q).SingleOrDefault()) .Where(dte => dte != null && dte.Properties.ContainsKey(indexPropertyName)) .GroupBy(dte => dte.Properties[indexPropertyName].PropertyAsObject, dte => Guid.Parse(dte.PartitionKey), equalityComparer) .Where(g => g.Count() > 1) .ToArray(); foreach (var collision in collisions) { JsonSerializer.Create().Serialize(sw, collision.Key); sw.Flush(); var guid = new Guid(MD5.Create().ComputeHash(collisionBytes, 0, (int)ms.Position)); ms.Position = 0; onCollision(guid, collision); } }); } }
/// <summary> /// Transforms the values of a column in an Azure table. The column may be a normal column or the RowKey column, but cannot be the PartitionKey column. /// The column to be transformed is specified using the following extended properties /// Extended Properties /// columnName - Name of the column to be transformed /// columnType - Data type of the column. Only supported types right now are: int32, bool, and string /// ifColumnValueMatches - The transformation is applied only if the contents of column value matches the specified value. /// replaceColumnValueWith - Replace the contents of the matched column value with the specified value. /// ifRowKeyContains - The transformation is applied only if the contents of row key contains the specified value. /// replaceRowKeySubStrWith - Replace the contents of the matched row key with the specified value to generate a new row key. /// rowKeyPrefixes - Rowkey prefixes of the rows in which the column transformation will be applied. This is optional and will identify the subset of rows to do this operation. /// You can specify columnName,columnType,ifColumnValueMatches,replaceColumnValueWith or ifRowKeyContains,replaceRowKeySubStrWith or both as they work on different column types /// Extended Properties Example /// "columnName": "IdentityProviderType", /// "columnType": "string", /// "ifColumnValueMatches": "Beihai", /// "replaceColumnValueWith": "AADS2S", /// "ifRowKeyContains": "Beihai", /// "replaceRowKeySubStrWith": "AADS2S" /// Activity Operation /// The activity iterates through all the rows from the input table with the matching rowKeyPrefixes, /// checks for the column, apply the column transformation if the column value match is found /// checks for the row key update, apply the row key transformation if the row key match is found /// runs a replace table operation in case of column transformation only /// runs a delete insert operation in case of row key transformation /// </summary> /// <param name="linkedServices">Linked services referenced by activity definition.</param> /// <param name="datasets">Datasets referenced by activity definition.</param> /// <param name="activity">Activity definition.</param> /// <param name="logger">Used to log messages during activity execution.</param> /// <returns>Activity state at the end of execution</returns> public IDictionary <string, string> Execute( IEnumerable <LinkedService> linkedServices, IEnumerable <Dataset> datasets, Microsoft.Azure.Management.DataFactories.Models.Activity activity, IActivityLogger logger) { DotNetActivity dotNetActivity = (DotNetActivity)activity.TypeProperties; IDictionary <string, string> extendedProperties = dotNetActivity.ExtendedProperties; logger.Write("Logging extended properties if any..."); foreach (KeyValuePair <string, string> entry in extendedProperties) { logger.Write("<key:{0}> <value:{1}>", entry.Key, entry.Value); } string[] rowKeyPrefixes = null; if (extendedProperties.ContainsKey("rowKeyPrefixes")) { rowKeyPrefixes = extendedProperties["rowKeyPrefixes"].Split(','); } bool hasColumnUpdate = false; string columnName = string.Empty, columnType = string.Empty, ifColumnValueMatches = string.Empty, replaceColumnValueWith = string.Empty; if (extendedProperties.ContainsKey("columnName")) { columnName = extendedProperties["columnName"]; columnType = extendedProperties["columnType"]; ifColumnValueMatches = extendedProperties["ifColumnValueMatches"]; replaceColumnValueWith = extendedProperties["replaceColumnValueWith"]; hasColumnUpdate = true; } bool hasRowKeyUpdate = false; string ifRowKeyContains = string.Empty, replaceRowKeySubStrWith = string.Empty; if (extendedProperties.ContainsKey("ifRowKeyContains")) { ifRowKeyContains = extendedProperties["ifRowKeyContains"]; replaceRowKeySubStrWith = extendedProperties["replaceRowKeySubStrWith"]; hasRowKeyUpdate = true; } AzureStorageLinkedService inputLinkedService; AzureTableDataset sourceTable; // For activities working on a single dataset, the first entry is the input dataset. // The activity.Inputs can have multiple datasets for building pipeline workflow dependencies. We can ignore the rest of the datasets Dataset inputDataset = datasets.Single(dataset => dataset.Name == activity.Inputs.First().Name); sourceTable = inputDataset.Properties.TypeProperties as AzureTableDataset; logger.Write("input table:{0}", sourceTable.TableName); inputLinkedService = linkedServices.First( ls => ls.Name == inputDataset.Properties.LinkedServiceName).Properties.TypeProperties as AzureStorageLinkedService; string inputConnectionString = inputLinkedService.ConnectionString; // create storage client for input. Pass the connection string. CloudStorageAccount inputStorageAccount = CloudStorageAccount.Parse(inputConnectionString); CloudTableClient inputTableClient = inputStorageAccount.CreateCloudTableClient(); CloudTable inputTable = inputTableClient.GetTableReference(sourceTable.TableName); long totalProcessedRecords = 0; long actualAffectedRecords = 0; TableContinuationToken tableContinuationToken = null; List <Task> tasks = new List <Task>(); do { var resultSegment = inputTable.ExecuteQuerySegmented(new TableQuery(), tableContinuationToken); tableContinuationToken = resultSegment.ContinuationToken; var partitionGroups = (from s in resultSegment.Results where (rowKeyPrefixes == null || rowKeyPrefixes.Length <= 0) ? true : this.IsMatch(s.RowKey, rowKeyPrefixes) select s).GroupBy(a => a.PartitionKey); foreach (IGrouping <string, DynamicTableEntity> g in partitionGroups) { TableBatchOperation batch = new TableBatchOperation(); foreach (DynamicTableEntity e in g.AsEnumerable()) { string cachedRowkey = e.RowKey; IDictionary <string, EntityProperty> cachedProperties = new Dictionary <string, EntityProperty>(); foreach (KeyValuePair <string, EntityProperty> p in e.Properties) { cachedProperties.Add(p); } bool recordUpdated = false, requiresDelete = false; if (hasColumnUpdate) { recordUpdated = this.ReplaceIfMatch(e, columnName, columnType, ifColumnValueMatches, replaceColumnValueWith); } if (hasRowKeyUpdate && e.RowKey.Contains(ifRowKeyContains)) { e.RowKey = e.RowKey.Replace(ifRowKeyContains, replaceRowKeySubStrWith); recordUpdated = true; requiresDelete = true; } if (recordUpdated) { if (!requiresDelete) { batch.Replace(e); } else { batch.Insert(e); batch.Delete(new DynamicTableEntity(e.PartitionKey, cachedRowkey, "*", cachedProperties)); } actualAffectedRecords++; logger.Write("<partition key:{0}>, <row key:{1}> added to batch", e.PartitionKey, e.RowKey); } } if (batch.Count > 0) { tasks.Add(inputTable.ExecuteBatchInChunkAsync(batch)); } logger.Write("Updated partition: {0}", g.Key); } totalProcessedRecords += resultSegment.Results.Count; logger.Write("Processed records count: {0}", totalProcessedRecords); logger.Write("Affected records count: {0}", actualAffectedRecords); }while (tableContinuationToken != null); Task.WaitAll(tasks.ToArray()); logger.Write("Updated {0} records", actualAffectedRecords); return(new Dictionary <string, string>()); }
/// <summary> /// Processes the queue message. /// </summary> /// <param name="message">The message.</param> /// <param name="blogTable">The blog table.</param> /// <param name="subscriberTable">The subscriber table.</param> public static void ProcessNewPostQueueMessage( [QueueTrigger(NewPostQueue)] string message, [Table(BlogTable)] CloudTable blogTable, [Table(SubscriberTable)] CloudTable subscriberTable) { try { Console.Out.WriteLine("New post message captured {0}", message); var sendgridUserName = ConfigurationManager.AppSettings[ApplicationConstants.SendgridUserName]; var sendgridPassword = ConfigurationManager.AppSettings[ApplicationConstants.SendgridPassword]; mailSystem = new SendgridMailClient(sendgridUserName, sendgridPassword); var operation = TableOperation.Retrieve(ApplicationConstants.BlogKey, message); var result = blogTable.Execute(operation, TableRequestOptions).Result as DynamicTableEntity; if (null == result) { Console.Error.WriteLine("Could not find record corresponding to RK {0}", message); return; } var title = result.Properties["Title"].StringValue; var postedDate = result.Properties["PostedDate"].DateTime; var bodySnippet = Routines.GeneratePreview(result.Properties["AutoIndexedElement_0_Body"].StringValue); var formattedUri = result.Properties["FormattedUri"].StringValue; //// Run paged queries to get subscribers. Console.Out.WriteLine("Going to get list of subscribers"); var query = (from record in subscriberTable.CreateQuery <DynamicTableEntity>() where record.PartitionKey == ApplicationConstants.SubscriberListKey && record.Properties["LastEmailIdentifier"].StringValue != result.RowKey && record.Properties["LastEmailIdentifier"].StringValue != string.Format("Faulted {0}", DateTime.UtcNow.Date) && record.Properties["IsVerified"].BooleanValue == true select record).Take(50).AsTableQuery(); TableContinuationToken token = null; do { var segment = subscriberTable.ExecuteQuerySegmented(query, token, TableRequestOptions); var batchUpdate = new TableBatchOperation(); if (null == segment || !segment.Any()) { Console.Out.WriteLine("No users found. Aborting current call."); return; } //// Compose Tuple of records of users. var userDetails = segment.Select( record => new Tuple <string, string, string>( record.Properties["FirstName"].StringValue, record.RowKey, record.Properties["VerificationString"].StringValue)).ToList(); Console.Out.WriteLine("Going to send mails to users"); if (SendNewPostMailsToUsers(userDetails, title, postedDate, formattedUri, bodySnippet)) { foreach (var record in segment) { record.Properties["LastEmailIdentifier"].StringValue = result.RowKey; batchUpdate.Add(TableOperation.InsertOrReplace(record)); } } else { foreach (var record in segment) { record.Properties["LastEmailIdentifier"].StringValue = string.Format( "Faulted {0}", DateTime.UtcNow.Date); batchUpdate.Add(TableOperation.InsertOrReplace(record)); } } subscriberTable.ExecuteBatch(batchUpdate, TableRequestOptions); token = segment.ContinuationToken; }while (token != null); Console.Out.WriteLine("Mails sent"); } catch (Exception exception) { Console.Error.WriteLine("Error at Time:{0} Message:{1}", DateTime.UtcNow, exception); throw; } }
public IEnumerable <T> RetrieveAllInPartition <T>(string partitionKey) where T : ITableEntity, new() { var query = new TableQuery <T>().Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey)); return(tableContext.ExecuteQuerySegmented <T>(query, null).Results); }