/// <summary> /// Registers the shardlet connection. /// </summary> /// <param name="shardlet">The shardlet.</param> /// <param name="spid">The spid.</param> public override void PublishShardletConnection(Shardlet shardlet, short spid) { var repository = new AzureShardletConnectionRepository(shardlet.ShardSetName); var azureShardletConnection = repository.Get(shardlet, spid); if (azureShardletConnection != null) { // updates the timestamp repository.Merge(azureShardletConnection); return; } azureShardletConnection = new AzureShardletConnection { Catalog = shardlet.Catalog, DistributionKey = shardlet.DistributionKey, ShardingKey = shardlet.ShardingKey, Spid = spid }; repository.Insert(azureShardletConnection); }
/// <summary> /// Gets the Shardlet by distribution key from either the pinned Shardlet list or by range lookup. /// </summary> /// <param name="shardSetName">Name of the shard set.</param> /// <param name="distributionKey">The distribution key.</param> /// <param name="isNew">if set to <c>true</c> the key is a new key and implementer should not look for existing ShardLet.</param> /// <returns>Shardlet.</returns> public override Shardlet GetShardletByDistributionKey(string shardSetName, long distributionKey, bool isNew = false) { var repository = new AzureShardletMapRepository(shardSetName); if (!isNew) { var azureShardlet = GetAzureShardlet(shardSetName, distributionKey); if (azureShardlet != null) { return(azureShardlet.ToFrameworkShardlet()); } } var shardlet = new Shardlet(); //todo: how to set the sharding key? var rangeShard = repository.Get(distributionKey); if (rangeShard == null) { return(shardlet); } shardlet.ServerInstanceName = rangeShard.ServerInstanceName; shardlet.Catalog = rangeShard.Catalog; shardlet.ShardSetName = shardSetName; shardlet.DistributionKey = distributionKey; //TODO: Determine if the published shard should have a status and derive it from that for full shard transitions. shardlet.Status = ShardletStatus.Active; return(shardlet); }
/// <summary> /// Deletes the rows associated with the collection of SPIDs for the specified shardlet. /// </summary> /// <param name="shardlet">The shardlet.</param> /// <param name="spids">The spids.</param> public void Delete(Shardlet shardlet, IEnumerable <short> spids) { var array = spids.ToArray(); var batches = Enumerable.Range(0, array.Count()).GroupBy(i => i / BatchSize, i => array[i]); var partitionKey = shardlet.DistributionKey.ToString(CultureInfo.InvariantCulture); foreach (var batch in batches) { var batchOperation = new TableBatchOperation(); foreach (var spid in batch) { var rowKey = AzureShardletConnection.GetRowKey(shardlet.Catalog, spid); var entity = new DynamicTableEntity { RowKey = rowKey, PartitionKey = partitionKey, ETag = "*" }; batchOperation.Delete(entity); } RetryPolicyFactory.GetDefaultAzureStorageRetryPolicy() .ExecuteAction(() => _table.ExecuteBatch(batchOperation)); foreach (var spid in batch) { AzureCache.Remove(GetCacheKey(_cacheType, partitionKey, AzureShardletConnection.GetRowKey(shardlet.Catalog, spid))); } } }
/// <summary> /// Gets the specified shardlet mapped to a specific spid. /// </summary> /// <param name="shardlet">The shardlet.</param> /// <param name="spid">The spid.</param> /// <returns>AzureShardletConnection.</returns> public AzureShardletConnection Get(Shardlet shardlet, int spid) { var partitionKey = shardlet.DistributionKey.ToString(CultureInfo.InvariantCulture); var rowKey = AzureShardletConnection.GetRowKey(shardlet.Catalog, spid); var cacheShardletConnection = AzureCache.Get <CacheShardletConnection>(GetCacheKey(_cacheType, partitionKey, rowKey)); if (cacheShardletConnection != null) { return(cacheShardletConnection.ToAzureShardletConnection()); } var condition1 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey); var condition2 = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey); var condition = TableQuery.CombineFilters(condition1, TableOperators.And, condition2); var query = new TableQuery <AzureShardletConnection>() .Where(condition); IEnumerable <AzureShardletConnection> result = null; RetryPolicyFactory.GetDefaultAzureStorageRetryPolicy() .ExecuteAction(() => result = _table.ExecuteQuery(query)); var azureShardletConnection = result.FirstOrDefault(); Cache(azureShardletConnection); return(azureShardletConnection); }
public void T4_Pin_Data_To_Shard() { var shardSetConfig = ShardSetConfig.LoadCurrent(TestShardSetName); var pointerShard = shardSetConfig.Shards.First(); var shardlet = Shardlet.Load(TestShardSetName, "1"); shardlet.MoveToShard(pointerShard, true, true, new Guid(), _queueAndUseWorkerRoles); }
private void DeleteAzureShardlet(string shardSetName, Shardlet shardlet) { var repository = new AzureShardletMapRepository(shardSetName); var azureShardlet = repository.Get(shardlet.DistributionKey); if (azureShardlet != null) { repository.Delete(azureShardlet); } }
public void Close() { if (State != ConnectionState.Closed && State != ConnectionState.Broken) { Shardlet.Disconnect(_shardlet, _spid); _sqlConnection.Close(); } _spid = 0; }
public void Open() { if (State == ConnectionState.Closed) { _sqlConnection.Open(); _spid = GetSpid(); Shardlet.Connect(_shardlet, _spid); } }
/// <summary> /// Terminates the connections to the shardlet. /// </summary> /// <param name="shardlet">The shardlet.</param> public override void TerminateConnections(Shardlet shardlet) { var repository = new AzureShardletConnectionRepository(shardlet.ShardSetName); // get all connections to the shardlet var spids = repository.Get(shardlet); // terminate all connections.... TerminateDatabaseConnections(shardlet, spids); // delete all connections from the shardlet repository repository.Delete(shardlet, spids); }
/// <summary> /// De-registers the shardlet connection. /// </summary> /// <param name="shardlet">The shardlet.</param> /// <param name="spid">The spid.</param> /// <returns>Microsoft.AzureCat.Patterns.DataElasticity.Models.ShardletStatus.</returns> /// <exception cref="System.NotImplementedException"></exception> public override void RemoveShardletConnection(Shardlet shardlet, short spid) { // todo: do we have to look it up? or can we just fail to delete? var repository = new AzureShardletConnectionRepository(shardlet.ShardSetName); var azureShardletConnection = repository.Get(shardlet, spid); if (azureShardletConnection == null) { return; } repository.Delete(azureShardletConnection); }
/// <summary> /// Initializes a new instance of the <see cref="ElasticSqlConnection"/> class. /// </summary> /// <param name="shardSetName">Name of the shard set.</param> /// <param name="shardingKey">The sharding key.</param> public ElasticSqlConnection(string shardSetName, string shardingKey) { _shardlet = Shardlet.Load(shardSetName, shardingKey); if (_shardlet == null) { throw new ElasticDataException(string.Format("The Data Elasticity cannot locate a pinned shardlet or shardlet range for this sharding key: {0}", shardingKey)); } if (_shardlet.Status == ShardletStatus.Moving) { throw new ElasticDataException(string.Format("The Data Elasticity is currently moving the shardlet with sharding key: {0}", shardingKey)); } _sqlConnection = new ReliableSqlConnection(_shardlet.ConnectionString).Current; }
/// <summary> /// Gets the SQL Server SPIDs currently connected to a specific shardlet. /// </summary> /// <param name="shardlet">The shardlet.</param> /// <returns>IEnumerable<System.Int32>.</returns> public IEnumerable <short> Get(Shardlet shardlet) { // todo - dynamic parameter var condition = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, shardlet.DistributionKey.ToString(CultureInfo.InvariantCulture)); var query = new TableQuery <AzureShardletConnection>() .Where(condition); IEnumerable <AzureShardletConnection> result = null; RetryPolicyFactory.GetDefaultAzureStorageRetryPolicy() .ExecuteAction(() => result = _table.ExecuteQuery(query)); return (result .Select(asc => (short)asc.Spid) .ToArray()); }
private static void TerminateDatabaseConnections(Shardlet shardlet, IEnumerable <short> spids) { // the kill statement cannot take a parameter, so // dynamic sql seems to be required const string sql = "If EXISTS(SELECT 1 from sys.dm_exec_sessions WHERE session_id = {0} AND login_time <= '{1}')" + " BEGIN" + " KILL {0};" + " END"; var connectionString = shardlet.ConnectionString; var connection = new ReliableSqlConnection(connectionString); var array = spids.ToArray(); const int batchSize = 25; var batches = Enumerable.Range(0, array.Count()).GroupBy(i => i / batchSize, i => array[i]); // todo: what should this be var loginTime = DateTime.UtcNow.ToString(CultureInfo.InvariantCulture); // send multiple kill states in batches using (connection) { connection.Open(); var command = new SqlCommand("", connection.Current); foreach (var batch in batches) { var builder = new StringBuilder(); foreach (var spid in batch) { builder.AppendLine(string.Format(sql, spid.ToString(CultureInfo.InvariantCulture), loginTime)); } command.CommandText = builder.ToString(); command.ExecuteNonQuery(); } } }
/// <summary> /// Gets the shardlet by sharding key. /// </summary> /// <param name="shardSetName">Name of the shard set.</param> /// <param name="shardingKey">The sharding key.</param> /// <param name="distributionKey">The distribution key.</param> /// <returns>Shardlet.</returns> public override Shardlet GetShardletByShardingKey(string shardSetName, string shardingKey, long distributionKey) { Shardlet shardlet; var repository = new AzureShardletMapRepository(shardSetName); var azureShardlet = repository.Get(distributionKey); if (azureShardlet == null) { var rangeShard = GetAzureRangeShard(shardSetName, distributionKey); if (rangeShard == null) { // would only arrive here if the shardmap is not yet // published or is published incorrectly return(null); } // add shardlet shardlet = new Shardlet { ServerInstanceName = rangeShard.ServerInstanceName, Catalog = rangeShard.Catalog, ShardSetName = shardSetName, DistributionKey = distributionKey, ShardingKey = shardingKey, Status = ShardletStatus.Active, }; SaveAzureShardlet(shardlet); } else { shardlet = azureShardlet.ToFrameworkShardlet(); } return(shardlet); }
/// <summary> /// Saves the azure Shardlet. /// </summary> /// <param name="shardlet">The shardlet.</param> private void SaveAzureShardlet(Shardlet shardlet) { var shardSetName = shardlet.ShardSetName; var distributionKey = shardlet.DistributionKey; var shardingKey = shardlet.ShardingKey; var status = shardlet.Status; var serverInstanceName = shardlet.ServerInstanceName; var catalog = shardlet.Catalog; var pinned = shardlet.Pinned; var repository = new AzureShardletMapRepository(shardSetName); var azureShardlet = repository.Get(distributionKey); if (azureShardlet == null) { azureShardlet = new AzureShardlet { ShardSetName = shardSetName, DistributionKey = distributionKey, ShardingKey = shardingKey, ServerInstanceName = serverInstanceName, Catalog = catalog, Status = status.ToString(), Pinned = pinned }; repository.Insert(azureShardlet); } else { azureShardlet.Catalog = catalog; azureShardlet.ServerInstanceName = serverInstanceName; azureShardlet.Status = status.ToString(); azureShardlet.Pinned = pinned; repository.Merge(azureShardlet); } }
/// <summary> /// Executes the command text against the shard connection in a non query fashion. /// </summary> /// <param name="commandText">The command text.</param> /// <param name="dataSet">The data set.</param> /// <returns>System.Int32 return value of the execution.</returns> public void ExecuteNonQuery(string commandText, string dataSet, string guid) { var shardlet = Shardlet.Load(_shardSetName, dataSet, guid); ExecuteNonQuery(commandText, new ReliableSqlConnection(shardlet.ConnectionString).Current); }
/// <summary> /// Executes the non query. /// </summary> /// Executes the command text against the shard connection in a non query fashion. /// <param name="shardingKey">The sharding key.</param> /// <returns>System.Int32 return value of the execution.</returns> public int ExecuteNonQuery(string commandText, string shardingKey) { var shardlet = Shardlet.Load(_shardSetName, shardingKey); return(ExecuteNonQuery(commandText, new ReliableSqlConnection(shardlet.ConnectionString).Current)); }
/// <summary> /// Executes the command text against the shard connection in a non query fashion. /// </summary> /// <param name="commandText">The command text.</param> /// <param name="distributionKey">The distribution key.</param> /// <returns>System.Int32 return value of the execution.</returns> public int ExecuteNonQuery(string commandText, long distributionKey) { var shardlet = Shardlet.Load(_shardSetName, distributionKey); return(ExecuteNonQuery(commandText, GetReliableConnection(shardlet))); }
private static SqlConnection GetReliableConnection(Shardlet shardlet) { return(new ReliableSqlConnection(shardlet.ConnectionString).Current); }
/// <summary> /// Executes the query. /// </summary> /// <param name="shardSetName">Name of the shard set.</param> /// <param name="commandText">The command text.</param> /// <param name="guid">The unique identifier.</param> /// <param name="dataSet">The data set.</param> /// <returns>DataTable.</returns> public DataTable ExecuteQuery(string shardSetName, string commandText, string guid, string dataSet) { var shardlet = Shardlet.Load("tpch", dataSet, guid); return(ExecuteQuery(commandText, new ReliableSqlConnection(shardlet.ConnectionString).Current)); }
//Return a result set /// <summary> /// Executes the query. /// </summary> /// <param name="shardSetName">Name of the shard set.</param> /// <param name="commandText">The command text.</param> /// <param name="distributionKey">The distribution key.</param> /// <returns>DataTable.</returns> public DataTable ExecuteQuery(string shardSetName, string commandText, long distributionKey) { var shardlet = Shardlet.Load(shardSetName, distributionKey); return(ExecuteQuery(commandText, new ReliableSqlConnection(shardlet.ConnectionString).Current)); }
/// <summary> /// Gets the connected SQL Server SPIDs for the shardlet. /// </summary> /// <param name="shardlet">The shardlet.</param> /// <returns>IEnumerable<System.Int32>.</returns> public IEnumerable <short> GetConnectedSpids(Shardlet shardlet) { var repository = new AzureShardletConnectionRepository(shardlet.ShardSetName); return(repository.Get(shardlet)); }
/// <summary> /// Publishes the Shardlet into the pinned Shardlet list. /// </summary> /// <param name="shardlet">The Shardlet.</param> public override void PublishShardlet(Shardlet shardlet) { SaveAzureShardlet(shardlet); }
/// <summary> /// Connects the specified shardlet in the connection map. /// </summary> /// <param name="shardlet">The shardlet.</param> /// <param name="spid">The spid.</param> public void Connect(Shardlet shardlet, short spid) { var shardletConnectionDriver = GetShardletConnectionDriver(shardlet.ShardSetName); shardletConnectionDriver.PublishShardletConnection(shardlet, spid); }
/// <summary> /// Disconnects the specified shardlet from the connection map /// </summary> /// <param name="shardlet">The shardlet.</param> /// <param name="spid">The spid.</param> public void Disconnect(Shardlet shardlet, short spid) { var shardletConnectionDriver = GetShardletConnectionDriver(shardlet.ShardSetName); shardletConnectionDriver.RemoveShardletConnection(shardlet, spid); }
/// <summary> /// Removes the Shardlet from the pinned Shardlet list. /// </summary> /// <param name="shardlet">The Shardlet.</param> public void RemoveShardlet(Shardlet shardlet) { DeleteAzureShardlet(shardlet.ShardSetName, shardlet); }