/// <summary>
/// Registers the shardlet connection.
/// </summary>
/// <param name="shardlet">The shardlet.</param>
/// <param name="spid">The spid.</param>
public override void PublishShardletConnection(Shardlet shardlet, short spid)
{
var repository = new AzureShardletConnectionRepository(shardlet.ShardSetName);
var azureShardletConnection = repository.Get(shardlet, spid);
if (azureShardletConnection != null)
{
// updates the timestamp
repository.Merge(azureShardletConnection);
return;
}
azureShardletConnection =
new AzureShardletConnection
{
Catalog = shardlet.Catalog,
DistributionKey = shardlet.DistributionKey,
ShardingKey = shardlet.ShardingKey,
Spid = spid
};
repository.Insert(azureShardletConnection);
}
/// <summary>
/// Gets the Shardlet by distribution key from either the pinned Shardlet list or by range lookup.
/// </summary>
/// <param name="shardSetName">Name of the shard set.</param>
/// <param name="distributionKey">The distribution key.</param>
/// <param name="isNew">if set to <c>true</c> the key is a new key and implementer should not look for existing ShardLet.</param>
/// <returns>Shardlet.</returns>
public override Shardlet GetShardletByDistributionKey(string shardSetName, long distributionKey,
bool isNew = false)
{
var repository = new AzureShardletMapRepository(shardSetName);
if (!isNew)
{
var azureShardlet = GetAzureShardlet(shardSetName, distributionKey);
if (azureShardlet != null)
{
return(azureShardlet.ToFrameworkShardlet());
}
}
var shardlet = new Shardlet();
//todo: how to set the sharding key?
var rangeShard = repository.Get(distributionKey);
if (rangeShard == null)
{
return(shardlet);
}
shardlet.ServerInstanceName = rangeShard.ServerInstanceName;
shardlet.Catalog = rangeShard.Catalog;
shardlet.ShardSetName = shardSetName;
shardlet.DistributionKey = distributionKey;
//TODO: Determine if the published shard should have a status and derive it from that for full shard transitions.
shardlet.Status = ShardletStatus.Active;
return(shardlet);
}
/// <summary>
/// Deletes the rows associated with the collection of SPIDs for the specified shardlet.
/// </summary>
/// <param name="shardlet">The shardlet.</param>
/// <param name="spids">The spids.</param>
public void Delete(Shardlet shardlet, IEnumerable <short> spids)
{
var array = spids.ToArray();
var batches = Enumerable.Range(0, array.Count()).GroupBy(i => i / BatchSize, i => array[i]);
var partitionKey = shardlet.DistributionKey.ToString(CultureInfo.InvariantCulture);
foreach (var batch in batches)
{
var batchOperation = new TableBatchOperation();
foreach (var spid in batch)
{
var rowKey = AzureShardletConnection.GetRowKey(shardlet.Catalog, spid);
var entity =
new DynamicTableEntity
{
RowKey = rowKey,
PartitionKey = partitionKey,
ETag = "*"
};
batchOperation.Delete(entity);
}
RetryPolicyFactory.GetDefaultAzureStorageRetryPolicy()
.ExecuteAction(() => _table.ExecuteBatch(batchOperation));
foreach (var spid in batch)
{
AzureCache.Remove(GetCacheKey(_cacheType,
partitionKey,
AzureShardletConnection.GetRowKey(shardlet.Catalog, spid)));
}
}
}
/// <summary>
/// Gets the specified shardlet mapped to a specific spid.
/// </summary>
/// <param name="shardlet">The shardlet.</param>
/// <param name="spid">The spid.</param>
/// <returns>AzureShardletConnection.</returns>
public AzureShardletConnection Get(Shardlet shardlet, int spid)
{
var partitionKey = shardlet.DistributionKey.ToString(CultureInfo.InvariantCulture);
var rowKey = AzureShardletConnection.GetRowKey(shardlet.Catalog, spid);
var cacheShardletConnection =
AzureCache.Get <CacheShardletConnection>(GetCacheKey(_cacheType, partitionKey, rowKey));
if (cacheShardletConnection != null)
{
return(cacheShardletConnection.ToAzureShardletConnection());
}
var condition1 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
var condition2 = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey);
var condition = TableQuery.CombineFilters(condition1, TableOperators.And, condition2);
var query =
new TableQuery <AzureShardletConnection>()
.Where(condition);
IEnumerable <AzureShardletConnection> result = null;
RetryPolicyFactory.GetDefaultAzureStorageRetryPolicy()
.ExecuteAction(() => result = _table.ExecuteQuery(query));
var azureShardletConnection = result.FirstOrDefault();
Cache(azureShardletConnection);
return(azureShardletConnection);
}
public void T4_Pin_Data_To_Shard()
{
var shardSetConfig = ShardSetConfig.LoadCurrent(TestShardSetName);
var pointerShard = shardSetConfig.Shards.First();
var shardlet = Shardlet.Load(TestShardSetName, "1");
shardlet.MoveToShard(pointerShard, true, true, new Guid(), _queueAndUseWorkerRoles);
}
private void DeleteAzureShardlet(string shardSetName, Shardlet shardlet)
{
var repository = new AzureShardletMapRepository(shardSetName);
var azureShardlet = repository.Get(shardlet.DistributionKey);
if (azureShardlet != null)
{
repository.Delete(azureShardlet);
}
}
public void Close()
{
if (State != ConnectionState.Closed && State != ConnectionState.Broken)
{
Shardlet.Disconnect(_shardlet, _spid);
_sqlConnection.Close();
}
_spid = 0;
}
public void Open()
{
if (State == ConnectionState.Closed)
{
_sqlConnection.Open();
_spid = GetSpid();
Shardlet.Connect(_shardlet, _spid);
}
}
/// <summary>
/// Terminates the connections to the shardlet.
/// </summary>
/// <param name="shardlet">The shardlet.</param>
public override void TerminateConnections(Shardlet shardlet)
{
var repository = new AzureShardletConnectionRepository(shardlet.ShardSetName);
// get all connections to the shardlet
var spids = repository.Get(shardlet);
// terminate all connections....
TerminateDatabaseConnections(shardlet, spids);
// delete all connections from the shardlet repository
repository.Delete(shardlet, spids);
}
/// <summary>
/// De-registers the shardlet connection.
/// </summary>
/// <param name="shardlet">The shardlet.</param>
/// <param name="spid">The spid.</param>
/// <returns>Microsoft.AzureCat.Patterns.DataElasticity.Models.ShardletStatus.</returns>
/// <exception cref="System.NotImplementedException"></exception>
public override void RemoveShardletConnection(Shardlet shardlet, short spid)
{
// todo: do we have to look it up? or can we just fail to delete?
var repository = new AzureShardletConnectionRepository(shardlet.ShardSetName);
var azureShardletConnection = repository.Get(shardlet, spid);
if (azureShardletConnection == null)
{
return;
}
repository.Delete(azureShardletConnection);
}
/// <summary>
/// Initializes a new instance of the <see cref="ElasticSqlConnection"/> class.
/// </summary>
/// <param name="shardSetName">Name of the shard set.</param>
/// <param name="shardingKey">The sharding key.</param>
public ElasticSqlConnection(string shardSetName, string shardingKey)
{
_shardlet = Shardlet.Load(shardSetName, shardingKey);
if (_shardlet == null)
{
throw new ElasticDataException(string.Format("The Data Elasticity cannot locate a pinned shardlet or shardlet range for this sharding key: {0}", shardingKey));
}
if (_shardlet.Status == ShardletStatus.Moving)
{
throw new ElasticDataException(string.Format("The Data Elasticity is currently moving the shardlet with sharding key: {0}", shardingKey));
}
_sqlConnection = new ReliableSqlConnection(_shardlet.ConnectionString).Current;
}
/// <summary>
/// Gets the SQL Server SPIDs currently connected to a specific shardlet.
/// </summary>
/// <param name="shardlet">The shardlet.</param>
/// <returns>IEnumerable<System.Int32>.</returns>
public IEnumerable <short> Get(Shardlet shardlet)
{
// todo - dynamic parameter
var condition = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal,
shardlet.DistributionKey.ToString(CultureInfo.InvariantCulture));
var query =
new TableQuery <AzureShardletConnection>()
.Where(condition);
IEnumerable <AzureShardletConnection> result = null;
RetryPolicyFactory.GetDefaultAzureStorageRetryPolicy()
.ExecuteAction(() => result = _table.ExecuteQuery(query));
return
(result
.Select(asc => (short)asc.Spid)
.ToArray());
}
private static void TerminateDatabaseConnections(Shardlet shardlet, IEnumerable <short> spids)
{
// the kill statement cannot take a parameter, so
// dynamic sql seems to be required
const string sql =
"If EXISTS(SELECT 1 from sys.dm_exec_sessions WHERE session_id = {0} AND login_time <= '{1}')"
+ " BEGIN"
+ " KILL {0};"
+ " END";
var connectionString = shardlet.ConnectionString;
var connection = new ReliableSqlConnection(connectionString);
var array = spids.ToArray();
const int batchSize = 25;
var batches = Enumerable.Range(0, array.Count()).GroupBy(i => i / batchSize, i => array[i]);
// todo: what should this be
var loginTime = DateTime.UtcNow.ToString(CultureInfo.InvariantCulture);
// send multiple kill states in batches
using (connection)
{
connection.Open();
var command = new SqlCommand("", connection.Current);
foreach (var batch in batches)
{
var builder = new StringBuilder();
foreach (var spid in batch)
{
builder.AppendLine(string.Format(sql, spid.ToString(CultureInfo.InvariantCulture), loginTime));
}
command.CommandText = builder.ToString();
command.ExecuteNonQuery();
}
}
}
/// <summary>
/// Gets the shardlet by sharding key.
/// </summary>
/// <param name="shardSetName">Name of the shard set.</param>
/// <param name="shardingKey">The sharding key.</param>
/// <param name="distributionKey">The distribution key.</param>
/// <returns>Shardlet.</returns>
public override Shardlet GetShardletByShardingKey(string shardSetName, string shardingKey, long distributionKey)
{
Shardlet shardlet;
var repository = new AzureShardletMapRepository(shardSetName);
var azureShardlet = repository.Get(distributionKey);
if (azureShardlet == null)
{
var rangeShard = GetAzureRangeShard(shardSetName, distributionKey);
if (rangeShard == null)
{
// would only arrive here if the shardmap is not yet
// published or is published incorrectly
return(null);
}
// add shardlet
shardlet =
new Shardlet
{
ServerInstanceName = rangeShard.ServerInstanceName,
Catalog = rangeShard.Catalog,
ShardSetName = shardSetName,
DistributionKey = distributionKey,
ShardingKey = shardingKey,
Status = ShardletStatus.Active,
};
SaveAzureShardlet(shardlet);
}
else
{
shardlet = azureShardlet.ToFrameworkShardlet();
}
return(shardlet);
}
/// <summary>
/// Saves the azure Shardlet.
/// </summary>
/// <param name="shardlet">The shardlet.</param>
private void SaveAzureShardlet(Shardlet shardlet)
{
var shardSetName = shardlet.ShardSetName;
var distributionKey = shardlet.DistributionKey;
var shardingKey = shardlet.ShardingKey;
var status = shardlet.Status;
var serverInstanceName = shardlet.ServerInstanceName;
var catalog = shardlet.Catalog;
var pinned = shardlet.Pinned;
var repository = new AzureShardletMapRepository(shardSetName);
var azureShardlet = repository.Get(distributionKey);
if (azureShardlet == null)
{
azureShardlet =
new AzureShardlet
{
ShardSetName = shardSetName,
DistributionKey = distributionKey,
ShardingKey = shardingKey,
ServerInstanceName = serverInstanceName,
Catalog = catalog,
Status = status.ToString(),
Pinned = pinned
};
repository.Insert(azureShardlet);
}
else
{
azureShardlet.Catalog = catalog;
azureShardlet.ServerInstanceName = serverInstanceName;
azureShardlet.Status = status.ToString();
azureShardlet.Pinned = pinned;
repository.Merge(azureShardlet);
}
}
/// <summary>
/// Executes the command text against the shard connection in a non query fashion.
/// </summary>
/// <param name="commandText">The command text.</param>
/// <param name="dataSet">The data set.</param>
/// <returns>System.Int32 return value of the execution.</returns>
public void ExecuteNonQuery(string commandText, string dataSet, string guid)
{
var shardlet = Shardlet.Load(_shardSetName, dataSet, guid);
ExecuteNonQuery(commandText, new ReliableSqlConnection(shardlet.ConnectionString).Current);
}
/// <summary>
/// Executes the non query.
/// </summary>
/// Executes the command text against the shard connection in a non query fashion.
/// <param name="shardingKey">The sharding key.</param>
/// <returns>System.Int32 return value of the execution.</returns>
public int ExecuteNonQuery(string commandText, string shardingKey)
{
var shardlet = Shardlet.Load(_shardSetName, shardingKey);
return(ExecuteNonQuery(commandText, new ReliableSqlConnection(shardlet.ConnectionString).Current));
}
/// <summary>
/// Executes the command text against the shard connection in a non query fashion.
/// </summary>
/// <param name="commandText">The command text.</param>
/// <param name="distributionKey">The distribution key.</param>
/// <returns>System.Int32 return value of the execution.</returns>
public int ExecuteNonQuery(string commandText, long distributionKey)
{
var shardlet = Shardlet.Load(_shardSetName, distributionKey);
return(ExecuteNonQuery(commandText, GetReliableConnection(shardlet)));
}
private static SqlConnection GetReliableConnection(Shardlet shardlet)
{
return(new ReliableSqlConnection(shardlet.ConnectionString).Current);
}
/// <summary>
/// Executes the query.
/// </summary>
/// <param name="shardSetName">Name of the shard set.</param>
/// <param name="commandText">The command text.</param>
/// <param name="guid">The unique identifier.</param>
/// <param name="dataSet">The data set.</param>
/// <returns>DataTable.</returns>
public DataTable ExecuteQuery(string shardSetName, string commandText, string guid, string dataSet)
{
var shardlet = Shardlet.Load("tpch", dataSet, guid);
return(ExecuteQuery(commandText, new ReliableSqlConnection(shardlet.ConnectionString).Current));
}
//Return a result set
/// <summary>
/// Executes the query.
/// </summary>
/// <param name="shardSetName">Name of the shard set.</param>
/// <param name="commandText">The command text.</param>
/// <param name="distributionKey">The distribution key.</param>
/// <returns>DataTable.</returns>
public DataTable ExecuteQuery(string shardSetName, string commandText, long distributionKey)
{
var shardlet = Shardlet.Load(shardSetName, distributionKey);
return(ExecuteQuery(commandText, new ReliableSqlConnection(shardlet.ConnectionString).Current));
}
/// <summary>
/// Gets the connected SQL Server SPIDs for the shardlet.
/// </summary>
/// <param name="shardlet">The shardlet.</param>
/// <returns>IEnumerable<System.Int32>.</returns>
public IEnumerable <short> GetConnectedSpids(Shardlet shardlet)
{
var repository = new AzureShardletConnectionRepository(shardlet.ShardSetName);
return(repository.Get(shardlet));
}
/// <summary>
/// Publishes the Shardlet into the pinned Shardlet list.
/// </summary>
/// <param name="shardlet">The Shardlet.</param>
public override void PublishShardlet(Shardlet shardlet)
{
SaveAzureShardlet(shardlet);
}
/// <summary>
/// Connects the specified shardlet in the connection map.
/// </summary>
/// <param name="shardlet">The shardlet.</param>
/// <param name="spid">The spid.</param>
public void Connect(Shardlet shardlet, short spid)
{
var shardletConnectionDriver = GetShardletConnectionDriver(shardlet.ShardSetName);
shardletConnectionDriver.PublishShardletConnection(shardlet, spid);
}
/// <summary>
/// Disconnects the specified shardlet from the connection map
/// </summary>
/// <param name="shardlet">The shardlet.</param>
/// <param name="spid">The spid.</param>
public void Disconnect(Shardlet shardlet, short spid)
{
var shardletConnectionDriver = GetShardletConnectionDriver(shardlet.ShardSetName);
shardletConnectionDriver.RemoveShardletConnection(shardlet, spid);
}
/// <summary>
/// Removes the Shardlet from the pinned Shardlet list.
/// </summary>
/// <param name="shardlet">The Shardlet.</param>
public void RemoveShardlet(Shardlet shardlet)
{
DeleteAzureShardlet(shardlet.ShardSetName, shardlet);
}
