/// <summary>
/// Deletes the specified azure shardlet connection.
/// </summary>
/// <param name="azureShardletConnection">The azure shardlet connection.</param>
public void Delete(AzureShardletConnection azureShardletConnection)
{
RetryPolicyFactory.GetDefaultAzureStorageRetryPolicy()
.ExecuteAction(() => _table.Execute(TableOperation.Delete(azureShardletConnection)));
AzureCache.Remove(GetCacheKey(_cacheType, azureShardletConnection));
}
/// <summary>
/// Gets the specified shardlet mapped to a specific spid.
/// </summary>
/// <param name="shardlet">The shardlet.</param>
/// <param name="spid">The spid.</param>
/// <returns>AzureShardletConnection.</returns>
public AzureShardletConnection Get(Shardlet shardlet, int spid)
{
var partitionKey = shardlet.DistributionKey.ToString(CultureInfo.InvariantCulture);
var rowKey = AzureShardletConnection.GetRowKey(shardlet.Catalog, spid);
var cacheShardletConnection =
AzureCache.Get <CacheShardletConnection>(GetCacheKey(_cacheType, partitionKey, rowKey));
if (cacheShardletConnection != null)
{
return(cacheShardletConnection.ToAzureShardletConnection());
}
var condition1 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
var condition2 = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey);
var condition = TableQuery.CombineFilters(condition1, TableOperators.And, condition2);
var query =
new TableQuery <AzureShardletConnection>()
.Where(condition);
IEnumerable <AzureShardletConnection> result = null;
RetryPolicyFactory.GetDefaultAzureStorageRetryPolicy()
.ExecuteAction(() => result = _table.ExecuteQuery(query));
var azureShardletConnection = result.FirstOrDefault();
Cache(azureShardletConnection);
return(azureShardletConnection);
}
/// <summary>
/// Gets the shardlet from the specified shard set name and distribution key.
/// </summary>
/// <param name="distributionKey">The distribution key.</param>
/// <returns>AzureShardlet.</returns>
public AzureShardlet Get(long distributionKey)
{
var cachedShardlet = AzureCache.Get <CacheShardlet>(GetCacheKey(_cacheType, _shardSetName, LongBasedRowKeyEntity.MakeRowKeyFromLong(distributionKey)));
if (cachedShardlet != null)
{
return(cachedShardlet.ToAzureShardlet());
}
var rowKey = LongBasedRowKeyEntity.MakeRowKeyFromLong(distributionKey);
var condition1 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, _shardSetName);
var condition2 = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey);
var condition = TableQuery.CombineFilters(condition1, TableOperators.And, condition2);
var query =
new TableQuery <AzureShardlet>()
.Where(condition);
IEnumerable <AzureShardlet> result = null;
RetryPolicyFactory.GetDefaultAzureStorageRetryPolicy()
.ExecuteAction(() => result = _table.ExecuteQuery(query));
var azureShardlet = result.FirstOrDefault();
Cache(azureShardlet);
return(azureShardlet);
}
/// <summary>
/// Deletes the rows associated with the collection of SPIDs for the specified shardlet.
/// </summary>
/// <param name="shardlet">The shardlet.</param>
/// <param name="spids">The spids.</param>
public void Delete(Shardlet shardlet, IEnumerable <short> spids)
{
var array = spids.ToArray();
var batches = Enumerable.Range(0, array.Count()).GroupBy(i => i / BatchSize, i => array[i]);
var partitionKey = shardlet.DistributionKey.ToString(CultureInfo.InvariantCulture);
foreach (var batch in batches)
{
var batchOperation = new TableBatchOperation();
foreach (var spid in batch)
{
var rowKey = AzureShardletConnection.GetRowKey(shardlet.Catalog, spid);
var entity =
new DynamicTableEntity
{
RowKey = rowKey,
PartitionKey = partitionKey,
ETag = "*"
};
batchOperation.Delete(entity);
}
RetryPolicyFactory.GetDefaultAzureStorageRetryPolicy()
.ExecuteAction(() => _table.ExecuteBatch(batchOperation));
foreach (var spid in batch)
{
AzureCache.Remove(GetCacheKey(_cacheType,
partitionKey,
AzureShardletConnection.GetRowKey(shardlet.Catalog, spid)));
}
}
}
public void ShouldGetIndividualShardlet()
{
//assemble
var driver = new AzureShardSetConnectionDriver();
var shardSetConfig = GetShardSetConfig();
InsertTestShardlets(driver, 10);
// act (should be in cache)
const string shardingKey = "1";
var distributionKey = GetDistributionKey(shardingKey);
var shardlet = driver.GetShardletByShardingKey(shardSetConfig.ShardSetName, shardingKey, distributionKey);
//assert
Assert.IsNotNull(shardlet);
Assert.AreEqual(shardingKey, shardlet.ShardingKey);
Assert.AreEqual(distributionKey, shardlet.DistributionKey);
// act (no caching)
AzureCache.Clear();
shardlet = driver.GetShardletByShardingKey(shardSetConfig.ShardSetName, shardingKey, distributionKey);
//assert
Assert.IsNotNull(shardlet);
Assert.AreEqual(shardingKey, shardlet.ShardingKey);
Assert.AreEqual(distributionKey, shardlet.DistributionKey);
}
public override void Run()
{
var context = GlobalHost.ConnectionManager.GetHubContext<PostHub>();
var azureCache = new AzureCache();
while (true)
{
var message=_cloudQueue.GetMessage();
if (message != null)
{
try
{
dynamic feedData = JObject.Parse(message.AsString);
var tableFeed = new Repository<Feed>();
var tablePost = new Repository<Post>();
var feed = tableFeed.GetByPartitionKeyAndRowKey(feedData.partitionKey.ToString(), feedData.rowKey.ToString());
var rssFeed = new QDFeedParser.HttpFeedFactory().CreateFeed(new Uri(feed.Url));
foreach (var data in rssFeed.Items)
{
try
{
var post = new Post
{
FeedId = feedData.rowKey,
PartitionKey = feedData.partitionKey,
RowKey = Regex.Replace(data.Title, @"[^\w]", ""),
Title = data.Title,
Content = data.Content,
Link = data.Link,
Timestamp = new DateTimeOffset(DateTime.Now),
};
tablePost.InsertOrReplace(post);
azureCache.Put<Post>(CacheKeys.Post(post.PartitionKey,post.RowKey), post);
}
catch (Exception)
{
//Log and do something with this error
}
}
_cloudQueue.DeleteMessage(message);
// The first time probably the stream is closed and you can receive a error.
// If you like avoid this issue you can use a retry policy
context.Clients.All.refreshPost(feed.Title);
}
catch (Exception ex)
{
//Log and do something with this error
}
}
else
{
Thread.Sleep(1000);
}
}
}
private void Cache(AzureShardlet azureShardlet)
{
if (azureShardlet == null)
{
return;
}
AzureCache.Put(GetCacheKey(_cacheType, azureShardlet), azureShardlet.ToCacheShardlet());
}
private void Cache(AzureShardletConnection azureShardletConnection)
{
if (azureShardletConnection == null)
{
return;
}
AzureCache.Put(GetCacheKey(_cacheType, azureShardletConnection),
azureShardletConnection.ToCacheShardletConnection());
}
/// <summary>
/// Deletes the shardlet connections older than the specified UTC Date and Time.
/// </summary>
/// <param name="utcDateTime">The UTC date time.</param>
public void Delete(DateTime utcDateTime)
{
var condition = TableQuery.GenerateFilterConditionForDate("Timestamp", QueryComparisons.LessThanOrEqual,
utcDateTime);
var query =
new TableQuery()
.Where(condition)
.Select(new[] { "PartitionKey", "RowKey" });
var result = _table.ExecuteQuery(query).ToArray();
var array = result.ToArray();
var batches = Enumerable.Range(0, array.Count()).GroupBy(i => i / BatchSize, i => array[i]);
foreach (var batch in batches)
{
var batchOperation = new TableBatchOperation();
foreach (var connection in batch)
{
var entity =
new DynamicTableEntity
{
RowKey = connection.RowKey,
PartitionKey = connection.PartitionKey,
ETag = "*"
};
batchOperation.Delete(entity);
}
RetryPolicyFactory.GetDefaultAzureStorageRetryPolicy()
.ExecuteAction(() => _table.ExecuteBatch(batchOperation));
foreach (var connection in batch)
{
AzureCache.Remove(GetCacheKey(_cacheType, connection.PartitionKey, connection.RowKey));
}
}
}
/// <summary>
/// Deletes the specified azure shardlet.
/// </summary>
/// <param name="azureShardlet">The azure shardlet.</param>
public void Delete(AzureShardlet azureShardlet)
{
_table.Execute(TableOperation.Delete(azureShardlet));
AzureCache.Remove(GetCacheKey(_cacheType, azureShardlet));
}
public void TestInitialize()
{
AzureShardSetConnectionDriver.InitializeAzureTables(_testShardSetName, true);
AzureShardSetConnectionDriver.InitializeAzureTables();
AzureCache.Clear();
}
public void T0_ClearCache()
{
AzureCache.Clear();
}
