public AzureOperationHandle AddValue(ScalableEntity value)
{
values.Add(value);
return(this);
}
public AzureDatabase(string storageAccountString, string tableName)
{
CloudStorageAccount account = CloudStorageAccount.Parse(storageAccountString);
qclient = account.CreateCloudQueueClient();
client = account.CreateCloudTableClient();
client.DefaultRequestOptions.PayloadFormat = TablePayloadFormat.JsonNoMetadata;
table = client.GetTableReference(tableName);
mthread = new Thread(async delegate() {
await table.CreateIfNotExistsAsync();
while (running)
{
evt.WaitOne();
evt.Reset();
Dictionary <ulong, List <AzureOperationHandle> > ops;
lock (evt)
{
ops = pendingOperations;
pendingOperations = new Dictionary <ulong, List <AzureOperationHandle> >();
}
List <Task> runningTasks = new List <Task>();
foreach (var shard in ops)
{
TableBatchOperation upserts = new TableBatchOperation();
TableBatchOperation deletions = new TableBatchOperation();
Dictionary <ScalableEntity, List <AzureOperationHandle> > retrieves = new Dictionary <ScalableEntity, List <AzureOperationHandle> >(EntityComparer.instance);
Dictionary <ByteRange, List <AzureOperationHandle> > rangeRetrieves = new Dictionary <ByteRange, List <AzureOperationHandle> >();
foreach (var op in shard.Value.Where(m => m.Type == OpType.Upsert || m.Type == OpType.Delete))
{
switch (op.Type)
{
case OpType.Upsert:
upserts.Add(TableOperation.InsertOrReplace(new AzureEntity()
{
PartitionKey = op.Entity.Partition.ToString(), RowKey = Uri.EscapeDataString(Convert.ToBase64String(op.Entity.Key)), Value = op.Entity.Value
}));
if (upserts.Count == 100)
{
runningTasks.Add(table.ExecuteBatchAsync(upserts));
upserts = new TableBatchOperation();
}
break;
case OpType.Delete:
deletions.Add(TableOperation.Delete(new AzureEntity()
{
PartitionKey = op.Entity.Partition.ToString(), RowKey = Uri.EscapeDataString(Convert.ToBase64String(op.Entity.Key)), Value = op.Entity.Value
}));
if (deletions.Count == 100)
{
runningTasks.Add(table.ExecuteBatchAsync(deletions));
deletions = new TableBatchOperation();
}
break;
}
}
Func <IEnumerable <string>, string> and = (q) => {
string query = null;
foreach (string er in q.Where(m => m != null))
{
if (query == null)
{
query = er;
}
else
{
query = TableQuery.CombineFilters(query, TableOperators.And, er);
}
}
return(query);
};
Func <IEnumerable <string>, string> or = (q) => {
string query = null;
foreach (string er in q.Where(m => m != null))
{
if (query == null)
{
query = er;
}
else
{
query = TableQuery.CombineFilters(query, TableOperators.Or, er);
}
}
return(query);
};
Func <Dictionary <ScalableEntity, List <AzureOperationHandle> >, Dictionary <ByteRange, List <AzureOperationHandle> >, TableContinuationToken, TableQuery <AzureEntity>, Task> runSegmentedQuery = null;
runSegmentedQuery = async(tableops, rangeops, token, compiledQuery) => {
if (compiledQuery == null)
{
string query = null;
foreach (var iable in tableops.Values.SelectMany(m => m).Where(m => m.Type == OpType.Retrieve))
{
if (query == null)
{
query = TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, iable.Entity.Partition.ToString()), TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, Uri.EscapeDataString(Convert.ToBase64String(iable.Entity.Key))));
}
else
{
query = TableQuery.CombineFilters(query, TableOperators.Or, TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, iable.Entity.Partition.ToString()), TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, Uri.EscapeDataString(Convert.ToBase64String(iable.Entity.Key)))));
}
}
foreach (var iable in rangeops.Values.SelectMany(m => m).Where(m => m.Type == OpType.RangeRetrieve))
{
string startQuery = null;
string endQuery = null;
if (iable.StartRange != null)
{
startQuery = TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThan, iable.StartRange.LinearHash().ToString()), TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, Uri.EscapeDataString(Convert.ToBase64String(iable.StartRange))));
}
if (iable.EndRange != null)
{
endQuery = TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThan, iable.EndRange.LinearHash().ToString()), TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, Uri.EscapeDataString(Convert.ToBase64String(iable.EndRange))));
}
query = or(new string[] { query, and(new string[] { startQuery, endQuery }) });
}
compiledQuery = new TableQuery <AzureEntity>().Where(query);
}
var segment = await table.ExecuteQuerySegmentedAsync(compiledQuery, token);
token = segment.ContinuationToken;
List <AzureOperationHandle> finished = new List <AzureOperationHandle>();
foreach (var iable in segment)
{
var ent = new ScalableEntity(Convert.FromBase64String(Uri.UnescapeDataString(iable.RowKey)), iable.Value);
if (tableops.ContainsKey(ent))
{
finished.AddRange(tableops[ent].Select(m => m.SetValue(ent.Value)));
}
if (rangeops.Any())
{
var range = new ByteRange(iable.Value, iable.Value);
if (rangeops.ContainsKey(range))
{
finished.AddRange(rangeops[range].Select(m => m.AddValue(new ScalableEntity(Convert.FromBase64String(Uri.UnescapeDataString(iable.RowKey)), iable.Value))));
}
}
}
//Combine callbacks for finished queries
var callbacks = finished.ToLookup(m => m.callback);
foreach (var iable in callbacks)
{
if (!iable.Key(iable.Where(m => m.Entity != null).Select(m => m.Entity).Union(iable.SelectMany(m => m.values), EntityComparer.instance)))
{
iable.AsParallel().ForAll(m => m.Type = OpType.Nop);
compiledQuery = null; //Re-compile query
}
}
if (token != null)
{
await runSegmentedQuery(tableops, rangeops, token, compiledQuery);
}
};
foreach (var op in shard.Value.Where(m => m.Type == OpType.Retrieve || m.Type == OpType.RangeRetrieve))
{
switch (op.Type)
{
case OpType.Retrieve:
if (!retrieves.ContainsKey(op.Entity))
{
retrieves.Add(op.Entity, new List <AzureOperationHandle>());
}
retrieves[op.Entity].Add(op);
if (retrieves.Count + rangeRetrieves.Count == 100)
{
runningTasks.Add(runSegmentedQuery(retrieves, rangeRetrieves, null, null));
retrieves = new Dictionary <ScalableEntity, List <AzureOperationHandle> >(EntityComparer.instance);
rangeRetrieves = new Dictionary <ByteRange, List <AzureOperationHandle> >();
}
break;
case OpType.RangeRetrieve:
var ranger = new ByteRange(op.StartRange, op.EndRange);
if (!rangeRetrieves.ContainsKey(ranger))
{
rangeRetrieves.Add(ranger, new List <AzureOperationHandle>());
}
rangeRetrieves[ranger].Add(op);
if (retrieves.Count + rangeRetrieves.Count == 100)
{
runningTasks.Add(runSegmentedQuery(retrieves, rangeRetrieves, null, null));
retrieves = new Dictionary <ScalableEntity, List <AzureOperationHandle> >(EntityComparer.instance);
rangeRetrieves = new Dictionary <ByteRange, List <AzureOperationHandle> >();
}
break;
}
}
foreach (var op in shard.Value.Where(m => m.Type == OpType.Nop))
{
op.Task.SetResult(op.Entity);
}
if (upserts.Any())
{
runningTasks.Add(table.ExecuteBatchAsync(upserts));
}
if (retrieves.Any() || rangeRetrieves.Any())
{
runningTasks.Add(runSegmentedQuery(retrieves, rangeRetrieves, null, null));
}
}
await Task.WhenAll(runningTasks);
ops.SelectMany(m => m.Value).AsParallel().ForAll(m => m.Task.SetResult(m.Entity));
}
});
mthread.Name = "AzureDB-webrunner";
mthread.Start();
}
public AzureOperationHandle(ScalableEntity entity, OpType type)
{
Entity = entity;
Task = new TaskCompletionSource <ScalableEntity>();
Type = type;
}
