public List <CustomerEntity> All()
{
TableQuery <CustomerEntity> tableQuery = new TableQuery <CustomerEntity>();
TableContinuationToken continuationToken = null;
var fullList = new List <CustomerEntity>();
do
{
TableQuerySegment <CustomerEntity> result = _azureTable.ExecuteQuerySegmented(tableQuery, continuationToken);
continuationToken = result.ContinuationToken;
var customerList = result as IList <CustomerEntity> ?? result.ToList();
fullList.AddRange(customerList);
} while (continuationToken != null);
return(fullList);
}
public IEnumerable <T> GetAll <T>() where T : TableEntity, new()
{
TableQuerySegment <T> querySegment = null;
while (querySegment == null || querySegment.ContinuationToken != null)
{
querySegment = m_table.ExecuteQuerySegmented(
new TableQuery <T>(),
querySegment != null ? querySegment.ContinuationToken : null
);
foreach (var entity in querySegment.Results)
{
yield return(entity);
}
}
}
public ActionResult Index()
{
TableQuery <ProductsFromTable> query = new TableQuery <ProductsFromTable>();
List <ProductsFromTable> products = new List <ProductsFromTable>();
TableContinuationToken token = null;
do
{
TableQuerySegment <ProductsFromTable> resultSegment = table.ExecuteQuerySegmented(query, token);
token = resultSegment.ContinuationToken;
foreach (ProductsFromTable product in resultSegment.Results)
{
products.Add(product);
}
} while (token != null);
return(View(products));
}
/// <summary>
/// Get all the records in the table
/// </summary>
/// <returns>All records</returns>
public IEnumerable <T> GetAllRecords()
{
var query = new TableQuery <T>();
var token = new TableContinuationToken();
var segment = CloudTable.ExecuteQuerySegmented(query, token);
while (token != null)
{
foreach (var result in segment)
{
yield return(result);
}
token = segment.ContinuationToken;
segment = CloudTable.ExecuteQuerySegmented(query, token);
}
}
static void QueryAllRecords()
{
TableQuery <employee> tq = new TableQuery <employee>().Where(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "Hyderabad"),
TableOperators.And,
TableQuery.GenerateFilterCondition("firstname", QueryComparisons.Equal, "rrr")
)
);
TableQuerySegment <employee> tqs = ct.ExecuteQuerySegmented <employee>(tq, new TableContinuationToken());
foreach (var data in tqs)
{
Console.WriteLine(data.firstname + " " + data.lastname + " " + data.PartitionKey);
}
}
static List <DynamicTableEntity> RetrieveAll(CloudTable table, IQueryable <DynamicTableEntity> query)
{
var entities = new List <DynamicTableEntity>();
TableContinuationToken token = null;
do
{
var page = query.Take(512);
var segment = table.ExecuteQuerySegmented((TableQuery <DynamicTableEntity>)page, token);
token = segment.ContinuationToken;
entities.AddRange(segment.Results);
}while (token != null);
return(entities);
}
private IEnumerable <DynamicTableEntity> ExecuteBalanceQuery(CloudTable table, TableQuery tableQuery, IEnumerable <int> pages)
{
pages = pages ?? new int[0];
var pagesEnumerator = pages.GetEnumerator();
TableContinuationToken continuation = null;
do
{
tableQuery.TakeCount = pagesEnumerator.MoveNext() ? (int?)pagesEnumerator.Current : null;
var segment = table.ExecuteQuerySegmented(tableQuery, continuation);
continuation = segment.ContinuationToken;
foreach (var entity in segment)
{
yield return(entity);
}
} while(continuation != null);
}
/// <summary>
/// Performs the <paramref name="action"/> on a <paramref name="table"/> filtered by <paramref name="filter"/>.
/// </summary>
/// <typeparam name="TEntity">The type of the entities in the <paramref name="table"/>.</typeparam>
/// <param name="table">The cloud table.</param>
/// <param name="action">The action to perform on the entities.</param>
/// <param name="filter">A filter for selecting the entities.</param>
public static void ProcessEntities <TEntity>(this CloudTable table, Action <IEnumerable <TEntity> > action, Expression <Func <TEntity, bool> > filter) where TEntity : ITableEntity, new()
{
TableQuerySegment <TEntity> segment = null;
while (segment == null || segment.ContinuationToken != null)
{
if (filter == null)
{
segment = table.ExecuteQuerySegmented(new TableQuery <TEntity>().Take(100), segment?.ContinuationToken);
}
else
{
var query = table.CreateQuery <TEntity>().Where(filter).Take(100).AsTableQuery();
segment = query.ExecuteSegmented(segment?.ContinuationToken);
}
action(segment.Results);
}
}
private void LoadFromPersistentStorage(CloudTable table)
{
TableContinuationToken?token = null;
var entities = new List <DynamicTableEntity>();
do
{
var queryResult = table.ExecuteQuerySegmented(new TableQuery(), token);
entities.AddRange(queryResult.Results);
token = queryResult.ContinuationToken;
} while (token != null);
foreach (var item in entities)
{
var vehicleJourneyAssignment = item.ToVehicleJourneyAssignment();
vehicleJourneyAssignmentCache.Put(vehicleJourneyAssignment);
}
}
//Returns a list of all Items in the items table
public static List <Item> GetItems()
{
CloudTable table = GetCloudTable();
TableQuery <Item> query = new TableQuery <Item>();
List <Item> itemsLst = new List <Item>();
TableContinuationToken token = null;
do
{
TableQuerySegment <Item> resultSegment = table.ExecuteQuerySegmented(query, token);
token = resultSegment.ContinuationToken;
foreach (Item item in resultSegment.Results)
{
itemsLst.Add(item);
}
} while (token != null);
return(itemsLst);
}
public Tuple <List <T>, TableContinuationToken> GetAllEntitiesByBatchs <T>(int maxNumOfElements,
TableContinuationToken continuationToken) where T : MyStateTableEntity, new()
{
// Create the CloudTable object that represents the "people" table.
CloudTable table = GetTableName <T>();
if (table == null)
{
return(null);
}
var tableQuery = new TableQuery <T>();
tableQuery.Take(maxNumOfElements);
var query = table.ExecuteQuerySegmented(
tableQuery, continuationToken);
return(new Tuple <List <T>, TableContinuationToken>(query.Results, query.ContinuationToken));
}
/// <summary>
/// Changes user validation and unsubscribe codes.
/// </summary>
/// <param name="subscriberTable">The subscriber table.</param>
public static void RefreshUserValidationStrings(CloudTable subscriberTable)
{
try
{
var query = (from record in subscriberTable.CreateQuery <DynamicTableEntity>()
where
record.PartitionKey == ApplicationConstants.SubscriberListKey &&
record.Properties["IsVerified"].BooleanValue == true
select record).Take(100).AsTableQuery();
TableContinuationToken token = null;
do
{
var segment = subscriberTable.ExecuteQuerySegmented(query, token, TableRequestOptions);
var batchUpdate = new TableBatchOperation();
if (null == segment || !segment.Any())
{
Console.Out.WriteLine("No users found. Aborting current call.");
}
else
{
//// Log users we are going to update.
foreach (var user in segment)
{
Console.Out.WriteLine("Updating {0}", user.Properties["Email"].StringValue);
user.Properties["VerificationString"].StringValue = Guid.NewGuid().ToString();
user.Properties["UnsubscribeString"].StringValue = Guid.NewGuid().ToString();
batchUpdate.Add(TableOperation.Replace(user));
}
Console.Out.WriteLine("Going to update user secrets");
subscriberTable.ExecuteBatch(batchUpdate, TableRequestOptions);
token = segment.ContinuationToken;
}
}while (token != null);
Console.Out.WriteLine("Secrets of all users updated.");
}
catch (Exception exception)
{
Console.Error.WriteLine("Error at Time:{0} Exception:{1}", DateTime.UtcNow, exception);
throw;
}
}
/// <summary>
/// Get the number of the records in the table
/// </summary>
/// <returns>The record count</returns>
public int GetRecordCount()
{
TableContinuationToken continuationToken = null;
var query = new TableQuery().Select(new List <string> {
"PartitionKey"
});
var recordCount = 0;
do
{
var items = CloudTable.ExecuteQuerySegmented(query, continuationToken);
continuationToken = items.ContinuationToken;
recordCount += items.Count();
} while (continuationToken != null);
return(recordCount);
}
//Returns the list of all departments in the departments table
public static List <Department> GetDepartments()
{
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference(DEP_TABLE_NAME);
TableQuery <Department> query = new TableQuery <Department>();
List <Department> departmentsLst = new List <Department>();
TableContinuationToken token = null;
do
{
TableQuerySegment <Department> resultSegment = table.ExecuteQuerySegmented(query, token);
token = resultSegment.ContinuationToken;
foreach (Department dept in resultSegment.Results)
{
departmentsLst.Add(dept);
}
} while (token != null);
return(departmentsLst);
}
private IEnumerable <StorageEntity> GetAllEntities(CloudTable table)
{
List <StorageEntity> result = new List <StorageEntity>();
// thanks to https://stackoverflow.com/a/48227035/2023653
var criteria = TableQuery.GenerateFilterCondition(nameof(StorageEntity.PartitionKey), QueryComparisons.Equal, PartitionKey);
var query = new TableQuery <StorageEntity>().Where(criteria);
TableContinuationToken continuationToken = null;
do
{
var results = table.ExecuteQuerySegmented(query, continuationToken);
foreach (var item in results)
{
result.Add(item);
}
continuationToken = results.ContinuationToken;
}while (continuationToken != null);
return(result);
}
public static async Task <IEnumerable <TEntity> > ExecuteQuery <TEntity>(
this CloudTable table,
TableQuery <TEntity> query,
CancellationToken cancellationToken)
where TEntity : ITableEntity, new()
{
var entities = new List <TEntity>();
TableContinuationToken continuation = null;
do
{
TableQuerySegment <TEntity> segment = await table
.ExecuteQuerySegmented(query, continuation, cancellationToken)
.ConfigureAwait(false);
entities.AddRange(segment);
continuation = segment.ContinuationToken;
}while (continuation != null);
return(entities);
}
private void ProcessEntities(CloudTable table,
Action <IEnumerable <DynamicTableEntity>, CloudTable> processor,
Expression <Func <DynamicTableEntity, bool> > filter)
{
TableQuerySegment <DynamicTableEntity> segment = null;
while (segment == null || segment.ContinuationToken != null)
{
if (filter == null)
{
segment = table.ExecuteQuerySegmented(new TableQuery().Take(100),
segment == null ? null : segment.ContinuationToken);
}
else
{
var query = table.CreateQuery <DynamicTableEntity>().Where(filter).Take(100).AsTableQuery();
segment = query.ExecuteSegmented(segment == null ? null : segment.ContinuationToken);
}
processor(segment.Results, table);
}
}
public async Task FlushAllPendingEvents(CancellationToken cancellationToken)
{
string filter = PendingEvent.FullScanFilter;
var query = new TableQuery <PendingEvent> {
FilterString = filter
};
TableContinuationToken continuation = null;
do
{
TableQuerySegment <PendingEvent> segment = await _eventTable
.ExecuteQuerySegmented(query, continuation, cancellationToken)
.ConfigureAwait(false);
foreach (string partition in segment.Select(e => e.PartitionKey).Distinct())
{
await Flush(partition, cancellationToken).ConfigureAwait(false);
}
continuation = segment.ContinuationToken;
}while (continuation != null);
}
//<summary>
//Select a random RowKey from a given PartitionKey
//</summary>
public string GetRandomRowKey(CloudTable table, string PartitionKey)
{
try
{
TableContinuationToken continuationToken = null;
var EntityRecords = new List <TableRecord>();
do
{
var rangeQueryResult = table
.ExecuteQuerySegmented(new TableQuery <TableRecord>()
.Where(TableQuery
.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, PartitionKey)), continuationToken);
EntityRecords.AddRange(rangeQueryResult.Results);
continuationToken = rangeQueryResult.ContinuationToken;
} while (continuationToken != null);
Dictionary <int, string> RowKeys = new Dictionary <int, string>();
int number = 1;
foreach (var Entity in EntityRecords)
{
RowKeys.Add(number, Entity.RowKey);
number++;
}
int subSize = RowKeys.Count;
Random rand = new Random();
const int MinValue = 1;
int rand_number = rand.Next(MinValue, subSize + 1);
string RandomRowKey = RowKeys[rand_number];
return(RandomRowKey);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Console.WriteLine(ex.Source);
Console.WriteLine(ex.StackTrace);
return(null);
}
}
public List <listcabackupconfig> GetBackups()
{
var continuationToken = new TableContinuationToken();
List <listcabackupconfig> config = new List <listcabackupconfig>();
do
{
var queryResult = table.ExecuteQuerySegmented(new TableQuery <listcabackupconfig>(), continuationToken);
foreach (var backupconfig in queryResult.Results)
{
string policy = "";
if (backupconfig.PartitionKey == "CABackupDaily")
{
policy = JObject.Parse(backupconfig.CABackup)["displayName"].ToString();
}
else if (backupconfig.PartitionKey == "CABackupChanges")
{
policy = JObject.Parse(backupconfig.CAOldValue)["displayName"].ToString();
}
config.Add(new listcabackupconfig
{
PartitionKey = backupconfig.PartitionKey,
RowKey = backupconfig.RowKey,
BackupDate = backupconfig.BackupDate,
Policy = policy,
BackupRestoreing = backupconfig.BackupRestoreing
});
}
continuationToken = queryResult.ContinuationToken;
} while (continuationToken != null);
return(config);
}
private List <AcrCallback> getLast(string streamId = TILOSHU_STREAMID, int limit = 1, int offset = 0)
{
CloudTable table = getTable();
TableQuery <AcrCallback> query = new TableQuery <AcrCallback>().Where(getPartitionFilter(streamId));
List <AcrCallback> playlist = new List <AcrCallback>();
TableContinuationToken token = null;
do
{
TableQuerySegment <AcrCallback> resultSegment = table.ExecuteQuerySegmented(query, token);
token = resultSegment.ContinuationToken;
foreach (AcrCallback customer in resultSegment.Results)
{
playlist.Add(customer);
}
} while (token != null && playlist.Count < (limit + offset));
var result = playlist.Skip(offset).Take(limit);
return(result.ToList());
}
private List <ZMTA_SALES> GetOrderById(string orderid)
{
List <ZMTA_SALES> orders = null;
InitializeCloudClient();
// Retrieve cloud table by tablename.
CloudTable table = cloudTableClient.GetTableReference("sapdatatable");
// Create table Query
TableQuery <ZMTA_SALES> query = new TableQuery <ZMTA_SALES>();
string partitionFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, orderid);
TableContinuationToken continuationToken = null;
var page = table.ExecuteQuerySegmented(query.Where(partitionFilter), continuationToken);
if (page.Results != null)
{
orders = new List <ZMTA_SALES>();
orders.AddRange(page.Results);
}
return(orders);
}
public override List <Event> Get(Guid aggregateId)
{
TableQuery <TableEvent> query = new TableQuery <TableEvent>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, aggregateId.ToString()));
List <Event> allEvents = new List <Event>();
TableContinuationToken token = new TableContinuationToken();
do
{
TableQuerySegment <TableEvent> events = _cloudTable.ExecuteQuerySegmented(query, token);
token = events.ContinuationToken;
foreach (var @event in events.Results)
{
allEvents.Add(ToEvent(@event));
}
} while (token != null);
return(allEvents
.OrderBy(e => e.Order)
.ToList());
}
public IActionResult Index()
{
var connectionstring = "DefaultEndpointsProtocol=https;AccountName=storageaccountkursu9c74;AccountKey=0UKdJ8IwdyLWE30jU43KGqnRu0n73sHORUHfVAE3XgVLHnZwbcez0MKpSOHs0IBNH6jUV9uBgA4gUSzq7LOfGQ==;EndpointSuffix=core.windows.net";
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(connectionstring);
CloudTableClient cloudTableClient = cloudStorageAccount.CreateCloudTableClient();
CloudTable cloudTable = cloudTableClient.GetTableReference("customers");
cloudTable.CreateIfNotExists();
Customer customer = new Customer("Skovby", Guid.NewGuid().ToString())
{
Email = "*****@*****.**",
FirstName = "Rune",
LastName = "Klan"
};
TableOperation operation = TableOperation.Insert(customer);
cloudTable.Execute(operation);
List <Customer> customers = new List <Customer>();
var hasMore = true;
while (hasMore)
{
TableContinuationToken token = null;
var queryResult = cloudTable.ExecuteQuerySegmented(new TableQuery <Customer>(), token);
customers.AddRange(queryResult.Results);
token = queryResult.ContinuationToken;
hasMore = token != null;
}
return(View(customers));
}
/// <summary>
/// Private extension method that performs an asynchronous query to an Azure table and provides progress callbacks while executing an Azure query asynchronously.
/// </summary>
/// <typeparam name="T">The type of table entity being returned.</typeparam>
/// <param name="table">The <see cref="CloudTable"/> to query.</param>
/// <param name="query">The <see cref="TableQuery"/> to send to Azure.</param>
/// <param name="ct">An optional <see cref="CancellationToken"/> that can be used to cancel the query.</param>
/// <param name="onProgress">An optional callback method to call as query results are coming in.</param>
/// <param name="onDone">An optional callback method to call when the query has completed.</param>
/// <returns>A <see cref="Task"/> that when complete contains the list of query results.</returns>
private static Task <IList <T> > ExecuteQueryAsync <T>(this CloudTable table, TableQuery <T> query, CancellationToken ct = default(CancellationToken), Action <IList <T> > onProgress = null, Action <IList <T>, bool> onDone = null) where T : ITableEntity, new()
{
return(Task.Run(() =>
{
var runningQuery = new TableQuery <T>()
{
FilterString = query.FilterString,
SelectColumns = query.SelectColumns
};
var items = new List <T>();
TableContinuationToken token = null;
do
{
runningQuery.TakeCount = query.TakeCount - items.Count;
Debug.WriteLine("Executing Query [" + query.FilterString + "] with Continuation Token [" + token?.NextTableName + token?.NextPartitionKey + token?.NextRowKey + "]");
TableQuerySegment <T> seg = table.ExecuteQuerySegmented <T>(runningQuery, token);
token = seg.ContinuationToken;
Debug.WriteLine("Query Returned " + seg.Results.Count + " Results, with new Continuation Token [" + seg.ContinuationToken?.NextTableName + seg.ContinuationToken?.NextPartitionKey + seg.ContinuationToken?.NextRowKey + "]");
items.AddRange(seg);
if (onProgress != null)
{
onProgress(seg.Results);
}
} while (token != null && !ct.IsCancellationRequested && (query.TakeCount == null || items.Count < query.TakeCount.Value));
if (onDone != null)
{
onDone(items, ct.IsCancellationRequested);
}
return (IList <T>)items;
}));
}
/// <summary>
/// Deletes a column from an Azure table.
/// The table for the column is same as the input table from the dataset
/// The column to be deleted is specified using the following extended properties
/// Extended Properties
/// columnName - Name of the column to be deleted
/// rowKeyPrefix - Rowkey prefix of the row from which the column will be deleted. This is optional and will identify the subset of rows to do this operation.
/// columnName is mandatory.
/// Extended Properties Example
/// "columnName": "UseDefault",
/// "rowKeyPrefix": "IdentityCredentialsObject:"
/// Activity Operation
/// The activity iterates through all the rows from the input table with the matching rowKeyPrefix,
/// checks for the column, removes the column if found and runs a replace table operation to replace the contents of
/// row/entity in the table.
/// </summary>
/// <param name="linkedServices">Linked services referenced by activity definition.</param>
/// <param name="datasets">Datasets referenced by activity definition.</param>
/// <param name="activity">Activity definition.</param>
/// <param name="logger">Used to log messages during activity execution.</param>
/// <returns>Activity state at the end of execution</returns>
public IDictionary <string, string> Execute(
IEnumerable <LinkedService> linkedServices,
IEnumerable <Dataset> datasets,
Activity activity,
IActivityLogger logger)
{
DotNetActivity dotNetActivity = (DotNetActivity)activity.TypeProperties;
IDictionary <string, string> extendedProperties = dotNetActivity.ExtendedProperties;
logger.Write("Logging extended properties if any...");
foreach (KeyValuePair <string, string> entry in extendedProperties)
{
logger.Write("<key:{0}> <value:{1}>", entry.Key, entry.Value);
}
if (!extendedProperties.ContainsKey("columnName"))
{
throw new ArgumentException("Column name is required", "columnName");
}
string columnName = extendedProperties["columnName"];
string rowKeyPrefix = string.Empty;
if (extendedProperties.ContainsKey("rowKeyPrefix"))
{
rowKeyPrefix = extendedProperties["rowKeyPrefix"];
}
AzureStorageLinkedService inputLinkedService;
AzureTableDataset sourceTable;
// For activities working on a single dataset, the first entry is the input dataset.
// The activity.Inputs can have multiple datasets for building pipeline workflow dependencies. We can ignore the rest of the datasets
Dataset inputDataset = datasets.Single(dataset => dataset.Name == activity.Inputs.First().Name);
sourceTable = inputDataset.Properties.TypeProperties as AzureTableDataset;
logger.Write("input table:{0}", sourceTable.TableName);
inputLinkedService = linkedServices.First(
ls =>
ls.Name ==
inputDataset.Properties.LinkedServiceName).Properties.TypeProperties
as AzureStorageLinkedService;
string inputConnectionString = inputLinkedService.ConnectionString;
// create storage client for input. Pass the connection string.
CloudStorageAccount inputStorageAccount = CloudStorageAccount.Parse(inputConnectionString);
CloudTableClient inputTableClient = inputStorageAccount.CreateCloudTableClient();
CloudTable inputTable = inputTableClient.GetTableReference(sourceTable.TableName);
long totalProcessedRecords = 0;
long actualAffectedRecords = 0;
TableContinuationToken tableContinuationToken = null;
List <Task> tasks = new List <Task>();
do
{
var resultSegment = inputTable.ExecuteQuerySegmented(new TableQuery(), tableContinuationToken);
tableContinuationToken = resultSegment.ContinuationToken;
var partitionGroups = (from s in resultSegment.Results
where string.IsNullOrWhiteSpace(rowKeyPrefix) ? true : s.RowKey.StartsWith(rowKeyPrefix)
select s).GroupBy(a => a.PartitionKey);
foreach (IGrouping <string, DynamicTableEntity> g in partitionGroups)
{
TableBatchOperation batch = new TableBatchOperation();
foreach (DynamicTableEntity e in g.AsEnumerable())
{
// If the columnName exist in the properties, then Remove it
if (e.Properties.ContainsKey(columnName))
{
e.Properties.Remove(columnName);
batch.Replace(e);
logger.Write("<partition key:{0}>, <row key:{1}> added to batch", e.PartitionKey, e.RowKey);
}
}
if (batch.Count > 0)
{
tasks.Add(inputTable.ExecuteBatchInChunkAsync(batch));
actualAffectedRecords += batch.Count;
}
}
totalProcessedRecords += resultSegment.Results.Count;
logger.Write("Processed records count: {0}", totalProcessedRecords);
logger.Write("Affected records count: {0}", actualAffectedRecords);
}while (tableContinuationToken != null);
// The batch operations complete when Task.WaitAll completes
Task.WaitAll(tasks.ToArray());
logger.Write("Deleted column from {0} records", actualAffectedRecords);
return(new Dictionary <string, string>());
}
public void SearchForDuplicates(Action <Guid, IEnumerable <Guid> > onCollision)
{
var query = new TableQuery
{
SelectColumns = columns
};
TableContinuationToken token = null;
var buffers = new List <IdHashBuffer>();
var buffer = new IdHashBuffer(InitialBufferSize);
buffers.Add(buffer);
var collisionBytes = new byte[4096];
var ms = new MemoryStream(collisionBytes);
var sw = new StreamWriter(ms);
do
{
var executeQuerySegmented = table.ExecuteQuerySegmented(query, token);
foreach (var dte in executeQuerySegmented.Results)
{
var id = Guid.Parse(dte.PartitionKey);
EntityProperty property;
if (dte.Properties.TryGetValue(indexPropertyName, out property))
{
var hash = hashingTransformer(property.PropertyAsObject);
if (buffer.TryWrite(id, hash) == false)
{
buffer = new IdHashBuffer(buffer.Size * 2);
buffers.Add(buffer);
if (buffer.TryWrite(id, hash) == false)
{
throw new OutOfMemoryException();
}
}
}
}
token = executeQuerySegmented.ContinuationToken;
} while (token != null);
foreach (var b in buffers)
{
b.Seal();
}
for (var i = 0; i < buffers.Count; i++)
{
var b = buffers[i];
b.FindHashCollisions(buffers.Skip(i), (hash, ids) =>
{
var collisions = ids.Select(id => new TableQuery
{
FilterString = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, id.ToString()),
SelectColumns = columns
})
.Select(q => table.ExecuteQuery(q).SingleOrDefault())
.Where(dte => dte != null && dte.Properties.ContainsKey(indexPropertyName))
.GroupBy(dte => dte.Properties[indexPropertyName].PropertyAsObject, dte => Guid.Parse(dte.PartitionKey), equalityComparer)
.Where(g => g.Count() > 1)
.ToArray();
foreach (var collision in collisions)
{
JsonSerializer.Create().Serialize(sw, collision.Key);
sw.Flush();
var guid = new Guid(MD5.Create().ComputeHash(collisionBytes, 0, (int)ms.Position));
ms.Position = 0;
onCollision(guid, collision);
}
});
}
}
/// <summary>
/// Transforms the values of a column in an Azure table. The column may be a normal column or the RowKey column, but cannot be the PartitionKey column.
/// The column to be transformed is specified using the following extended properties
/// Extended Properties
/// columnName - Name of the column to be transformed
/// columnType - Data type of the column. Only supported types right now are: int32, bool, and string
/// ifColumnValueMatches - The transformation is applied only if the contents of column value matches the specified value.
/// replaceColumnValueWith - Replace the contents of the matched column value with the specified value.
/// ifRowKeyContains - The transformation is applied only if the contents of row key contains the specified value.
/// replaceRowKeySubStrWith - Replace the contents of the matched row key with the specified value to generate a new row key.
/// rowKeyPrefixes - Rowkey prefixes of the rows in which the column transformation will be applied. This is optional and will identify the subset of rows to do this operation.
/// You can specify columnName,columnType,ifColumnValueMatches,replaceColumnValueWith or ifRowKeyContains,replaceRowKeySubStrWith or both as they work on different column types
/// Extended Properties Example
/// "columnName": "IdentityProviderType",
/// "columnType": "string",
/// "ifColumnValueMatches": "Beihai",
/// "replaceColumnValueWith": "AADS2S",
/// "ifRowKeyContains": "Beihai",
/// "replaceRowKeySubStrWith": "AADS2S"
/// Activity Operation
/// The activity iterates through all the rows from the input table with the matching rowKeyPrefixes,
/// checks for the column, apply the column transformation if the column value match is found
/// checks for the row key update, apply the row key transformation if the row key match is found
/// runs a replace table operation in case of column transformation only
/// runs a delete insert operation in case of row key transformation
/// </summary>
/// <param name="linkedServices">Linked services referenced by activity definition.</param>
/// <param name="datasets">Datasets referenced by activity definition.</param>
/// <param name="activity">Activity definition.</param>
/// <param name="logger">Used to log messages during activity execution.</param>
/// <returns>Activity state at the end of execution</returns>
public IDictionary <string, string> Execute(
IEnumerable <LinkedService> linkedServices,
IEnumerable <Dataset> datasets,
Microsoft.Azure.Management.DataFactories.Models.Activity activity,
IActivityLogger logger)
{
DotNetActivity dotNetActivity = (DotNetActivity)activity.TypeProperties;
IDictionary <string, string> extendedProperties = dotNetActivity.ExtendedProperties;
logger.Write("Logging extended properties if any...");
foreach (KeyValuePair <string, string> entry in extendedProperties)
{
logger.Write("<key:{0}> <value:{1}>", entry.Key, entry.Value);
}
string[] rowKeyPrefixes = null;
if (extendedProperties.ContainsKey("rowKeyPrefixes"))
{
rowKeyPrefixes = extendedProperties["rowKeyPrefixes"].Split(',');
}
bool hasColumnUpdate = false;
string columnName = string.Empty, columnType = string.Empty, ifColumnValueMatches = string.Empty, replaceColumnValueWith = string.Empty;
if (extendedProperties.ContainsKey("columnName"))
{
columnName = extendedProperties["columnName"];
columnType = extendedProperties["columnType"];
ifColumnValueMatches = extendedProperties["ifColumnValueMatches"];
replaceColumnValueWith = extendedProperties["replaceColumnValueWith"];
hasColumnUpdate = true;
}
bool hasRowKeyUpdate = false;
string ifRowKeyContains = string.Empty, replaceRowKeySubStrWith = string.Empty;
if (extendedProperties.ContainsKey("ifRowKeyContains"))
{
ifRowKeyContains = extendedProperties["ifRowKeyContains"];
replaceRowKeySubStrWith = extendedProperties["replaceRowKeySubStrWith"];
hasRowKeyUpdate = true;
}
AzureStorageLinkedService inputLinkedService;
AzureTableDataset sourceTable;
// For activities working on a single dataset, the first entry is the input dataset.
// The activity.Inputs can have multiple datasets for building pipeline workflow dependencies. We can ignore the rest of the datasets
Dataset inputDataset = datasets.Single(dataset => dataset.Name == activity.Inputs.First().Name);
sourceTable = inputDataset.Properties.TypeProperties as AzureTableDataset;
logger.Write("input table:{0}", sourceTable.TableName);
inputLinkedService = linkedServices.First(
ls =>
ls.Name ==
inputDataset.Properties.LinkedServiceName).Properties.TypeProperties
as AzureStorageLinkedService;
string inputConnectionString = inputLinkedService.ConnectionString;
// create storage client for input. Pass the connection string.
CloudStorageAccount inputStorageAccount = CloudStorageAccount.Parse(inputConnectionString);
CloudTableClient inputTableClient = inputStorageAccount.CreateCloudTableClient();
CloudTable inputTable = inputTableClient.GetTableReference(sourceTable.TableName);
long totalProcessedRecords = 0;
long actualAffectedRecords = 0;
TableContinuationToken tableContinuationToken = null;
List <Task> tasks = new List <Task>();
do
{
var resultSegment = inputTable.ExecuteQuerySegmented(new TableQuery(), tableContinuationToken);
tableContinuationToken = resultSegment.ContinuationToken;
var partitionGroups = (from s in resultSegment.Results
where (rowKeyPrefixes == null || rowKeyPrefixes.Length <= 0) ? true : this.IsMatch(s.RowKey, rowKeyPrefixes)
select s).GroupBy(a => a.PartitionKey);
foreach (IGrouping <string, DynamicTableEntity> g in partitionGroups)
{
TableBatchOperation batch = new TableBatchOperation();
foreach (DynamicTableEntity e in g.AsEnumerable())
{
string cachedRowkey = e.RowKey;
IDictionary <string, EntityProperty> cachedProperties = new Dictionary <string, EntityProperty>();
foreach (KeyValuePair <string, EntityProperty> p in e.Properties)
{
cachedProperties.Add(p);
}
bool recordUpdated = false, requiresDelete = false;
if (hasColumnUpdate)
{
recordUpdated = this.ReplaceIfMatch(e, columnName, columnType, ifColumnValueMatches, replaceColumnValueWith);
}
if (hasRowKeyUpdate && e.RowKey.Contains(ifRowKeyContains))
{
e.RowKey = e.RowKey.Replace(ifRowKeyContains, replaceRowKeySubStrWith);
recordUpdated = true;
requiresDelete = true;
}
if (recordUpdated)
{
if (!requiresDelete)
{
batch.Replace(e);
}
else
{
batch.Insert(e);
batch.Delete(new DynamicTableEntity(e.PartitionKey, cachedRowkey, "*", cachedProperties));
}
actualAffectedRecords++;
logger.Write("<partition key:{0}>, <row key:{1}> added to batch", e.PartitionKey, e.RowKey);
}
}
if (batch.Count > 0)
{
tasks.Add(inputTable.ExecuteBatchInChunkAsync(batch));
}
logger.Write("Updated partition: {0}", g.Key);
}
totalProcessedRecords += resultSegment.Results.Count;
logger.Write("Processed records count: {0}", totalProcessedRecords);
logger.Write("Affected records count: {0}", actualAffectedRecords);
}while (tableContinuationToken != null);
Task.WaitAll(tasks.ToArray());
logger.Write("Updated {0} records", actualAffectedRecords);
return(new Dictionary <string, string>());
}
/// <summary>
/// Processes the queue message.
/// </summary>
/// <param name="message">The message.</param>
/// <param name="blogTable">The blog table.</param>
/// <param name="subscriberTable">The subscriber table.</param>
public static void ProcessNewPostQueueMessage(
[QueueTrigger(NewPostQueue)] string message,
[Table(BlogTable)] CloudTable blogTable,
[Table(SubscriberTable)] CloudTable subscriberTable)
{
try
{
Console.Out.WriteLine("New post message captured {0}", message);
var sendgridUserName = ConfigurationManager.AppSettings[ApplicationConstants.SendgridUserName];
var sendgridPassword = ConfigurationManager.AppSettings[ApplicationConstants.SendgridPassword];
mailSystem = new SendgridMailClient(sendgridUserName, sendgridPassword);
var operation = TableOperation.Retrieve(ApplicationConstants.BlogKey, message);
var result = blogTable.Execute(operation, TableRequestOptions).Result as DynamicTableEntity;
if (null == result)
{
Console.Error.WriteLine("Could not find record corresponding to RK {0}", message);
return;
}
var title = result.Properties["Title"].StringValue;
var postedDate = result.Properties["PostedDate"].DateTime;
var bodySnippet = Routines.GeneratePreview(result.Properties["AutoIndexedElement_0_Body"].StringValue);
var formattedUri = result.Properties["FormattedUri"].StringValue;
//// Run paged queries to get subscribers.
Console.Out.WriteLine("Going to get list of subscribers");
var query = (from record in subscriberTable.CreateQuery <DynamicTableEntity>()
where
record.PartitionKey == ApplicationConstants.SubscriberListKey &&
record.Properties["LastEmailIdentifier"].StringValue != result.RowKey &&
record.Properties["LastEmailIdentifier"].StringValue
!= string.Format("Faulted {0}", DateTime.UtcNow.Date) &&
record.Properties["IsVerified"].BooleanValue == true
select record).Take(50).AsTableQuery();
TableContinuationToken token = null;
do
{
var segment = subscriberTable.ExecuteQuerySegmented(query, token, TableRequestOptions);
var batchUpdate = new TableBatchOperation();
if (null == segment || !segment.Any())
{
Console.Out.WriteLine("No users found. Aborting current call.");
return;
}
//// Compose Tuple of records of users.
var userDetails =
segment.Select(
record =>
new Tuple <string, string, string>(
record.Properties["FirstName"].StringValue,
record.RowKey,
record.Properties["VerificationString"].StringValue)).ToList();
Console.Out.WriteLine("Going to send mails to users");
if (SendNewPostMailsToUsers(userDetails, title, postedDate, formattedUri, bodySnippet))
{
foreach (var record in segment)
{
record.Properties["LastEmailIdentifier"].StringValue = result.RowKey;
batchUpdate.Add(TableOperation.InsertOrReplace(record));
}
}
else
{
foreach (var record in segment)
{
record.Properties["LastEmailIdentifier"].StringValue = string.Format(
"Faulted {0}",
DateTime.UtcNow.Date);
batchUpdate.Add(TableOperation.InsertOrReplace(record));
}
}
subscriberTable.ExecuteBatch(batchUpdate, TableRequestOptions);
token = segment.ContinuationToken;
}while (token != null);
Console.Out.WriteLine("Mails sent");
}
catch (Exception exception)
{
Console.Error.WriteLine("Error at Time:{0} Message:{1}", DateTime.UtcNow, exception);
throw;
}
}
public IEnumerable <T> RetrieveAllInPartition <T>(string partitionKey) where T : ITableEntity, new()
{
var query = new TableQuery <T>().Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey));
return(tableContext.ExecuteQuerySegmented <T>(query, null).Results);
}