public void LoadTestMarkAllShardsAsOnlineInListShardMap()
{
try
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
ListShardMap <int> lsm = smm.GetListShardMap <int>(ShardMapManagerLoadTests.s_listShardMapName);
Assert.IsNotNull(lsm);
foreach (Shard s in lsm.GetShards())
{
if (s.Status == ShardStatus.Offline)
{
lsm.UpdateShard(s,
new ShardUpdate
{
Status = ShardStatus.Online
});
}
}
}
catch (ShardManagementException sme)
{
Debug.WriteLine("Exception caught: {0}", sme.Message);
}
}
private static Shard CreateOrGetEmptyShard(ListShardMap <int> shardMap)
{
// Get an empty shard if one already exists, otherwise create a new one
Shard shard = FindEmptyShard(shardMap);
if (shard == null)
{
// No empty shard exists, so create one
// Choose the shard name
string databaseName = string.Format(ShardNameFormat, shardMap.GetShards().Count());
// Only create the database if it doesn't already exist. It might already exist if
// we tried to create it previously but hit a transient fault.
if (!SqlDatabaseUtils.DatabaseExists(Configuration.ShardMapManagerServerName, databaseName))
{
SqlDatabaseUtils.CreateDatabase(Configuration.ShardMapManagerServerName, databaseName);
}
// Create schema and populate reference data on that database
// The initialize script must be idempotent, in case it was already run on this database
// and we failed to add it to the shard map previously
SqlDatabaseUtils.ExecuteSqlScript(
Configuration.ShardMapManagerServerName, databaseName, InitializeShardScriptFile);
// Add it to the shard map
ShardLocation shardLocation = new ShardLocation(Configuration.ShardMapManagerServerName, databaseName);
shard = ShardManagementUtils.CreateOrGetShard(shardMap, shardLocation);
}
return(shard);
}
public void LoadTestRemoveShardFromListShardMap()
{
try
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
ListShardMap <int> lsm = smm.GetListShardMap <int>(ShardMapManagerLoadTests.s_listShardMapName);
Assert.IsNotNull(lsm);
List <Shard> existingShards = lsm.GetShards().ToList();
if (existingShards.Count == 0)
{
return;
}
// If there is already a shard marked as offline, chose that one to delete.
// This can happend if earlier remove operation was terminated for some reason - ex. killing connections.
Shard offlineShard = existingShards.Find(e => e.Status == ShardStatus.Offline);
if (offlineShard == null)
{
offlineShard = existingShards[_r.Next(existingShards.Count)];
// First mark shard as offline so that other test threads will not add new mappings to it.
offlineShard = lsm.UpdateShard(offlineShard,
new ShardUpdate
{
Status = ShardStatus.Offline
});
}
Debug.WriteLine("Trying to remove shard at location {0}", offlineShard.Location);
PointMappingUpdate pu = new PointMappingUpdate();
pu.Status = MappingStatus.Offline;
// Remove all mappings from this shard for given shard map.
foreach (PointMapping <int> p in lsm.GetMappings(offlineShard))
{
PointMapping <int> mappingToDelete = lsm.UpdateMapping(p, pu);
lsm.DeleteMapping(mappingToDelete);
}
// Shard object is changed as mappings are removed, get it again.
Shard deleteShard = lsm.GetShard(offlineShard.Location);
// now remove shard.
lsm.DeleteShard(deleteShard);
Debug.WriteLine("Removed shard at location {0} from shard map {1}", deleteShard.Location, lsm);
}
catch (ShardManagementException sme)
{
Debug.WriteLine("Exception caught: {0}", sme.Message);
}
}
public IEnumerable <string> GetAllShards()
{
string prefix = $"{_shardingConfig.ShardPrefix}.";
return(_shardMap.GetShards().Select(s =>
s.Location.Database.Split(new string[] { prefix },
StringSplitOptions.RemoveEmptyEntries).FirstOrDefault()));
}
public bool DeleteShardListMap()
{
try
{
if (ShardMapManager == null)
{
return(false);
}
Shard tempShard = null;
PointMapping <long> tempMapping = null;
if (ShardMapManager.TryGetListShardMap(mapManagerName, out lsm))
{
foreach (Shard shard in lsm.GetShards())
{
// delete all mappings
var allMappings = lsm.GetMappings(shard);
for (int i = 0; i < allMappings.Count; i++)
{
lsm.MarkMappingOffline(allMappings[i]);
if (lsm.TryGetMappingForKey(allMappings[i].Value, out tempMapping))
{
lsm.DeleteMapping(tempMapping);
}
}
// delete shard
if (lsm.TryGetShard(shard.Location, out tempShard))
{
lsm.DeleteShard(tempShard);
}
}
}
// clear shard map manager
if (ShardMapManager.TryGetListShardMap(mapManagerName, out lsm))
{
ShardMapManager.DeleteShardMap(lsm);
}
return(true);
}
catch { return(false); }
}
/// <summary>
/// Writes the shard map's state to the console.
/// </summary>
private static void PrintShardMapState()
{
Console.WriteLine("Current Shard Map state:");
ListShardMap <int> shardMap = TryGetShardMap();
if (shardMap == null)
{
return;
}
// Get all shards
IEnumerable <Shard> allShards = shardMap.GetShards();
// Get all mappings, grouped by the shard that they are on. We do this all in one go to minimise round trips.
//ILookup<Shard, RangeMapping<int>> mappingsGroupedByShard = shardMap.GetMappings().ToLookup(m => m.Shard);
var mappingsGroupedByShard = shardMap.GetMappings().ToLookup(m => m.Shard);
if (allShards.Any())
{
// The shard map contains some shards, so for each shard (sorted by database name)
// write out the mappings for that shard
foreach (Shard shard in shardMap.GetShards().OrderBy(s => s.Location.Database))
{
var mappingsOnThisShard = mappingsGroupedByShard[shard];
if (mappingsOnThisShard.Any())
{
string mappingsString = string.Join(", ", mappingsOnThisShard.Select(m => m.Value));
Console.WriteLine("\t{0} contains key range {1}", shard.Location.Database, mappingsString);
}
else
{
Console.WriteLine("\t{0} contains no key ranges.", shard.Location.Database);
}
}
}
else
{
Console.WriteLine("\tShard Map contains no shards");
}
}
private Shard FindEmptyShard(ListShardMap <int> shardMap)
{
// Get all shards in the shard map
IEnumerable <Shard> allShards = shardMap.GetShards();
// Get all mappings in the shard map
IEnumerable <PointMapping <int> > allMappings = shardMap.GetMappings();
// Determine which shards have mappings
HashSet <Shard> shardsWithMappings = new HashSet <Shard>(allMappings.Select(m => m.Shard));
// Get the first shard (ordered by name) that has no mappings, if it exists
return(allShards.OrderBy(s => s.Location.Database).FirstOrDefault(s => !shardsWithMappings.Contains(s)));
}
/// <summary>
/// Drops all shards and the shard map manager database (if it exists).
/// </summary>
private static void DropAll()
{
ListShardMap <int> shardMap = TryGetShardMap();
if (shardMap != null)
{
// Drop shards
foreach (Shard shard in shardMap.GetShards())
{
SqlDatabaseUtils.DropDatabase(shard.Location.DataSource, shard.Location.Database);
}
}
if (SqlDatabaseUtils.DatabaseExists(Configuration.ShardMapManagerServerName, Configuration.ShardMapManagerDatabaseName))
{
// Drop shard map manager database
SqlDatabaseUtils.DropDatabase(Configuration.ShardMapManagerServerName, Configuration.ShardMapManagerDatabaseName);
}
// Since we just dropped the shard map manager database, this shardMapManager reference is now non-functional.
// So set it to null so that the program knows that the shard map manager is gone.
s_shardMapManager = null;
}
/// <summary>
/// Creates a shard map manager, creates a shard map, and creates a shard
/// with a mapping for the full range of 32-bit integers.
/// </summary>
private static void CreateShardMapManagerAndShard()
{
if (s_shardMapManager != null)
{
ConsoleUtils.WriteWarning("Shard Map Manager already exists");
return;
}
// Create shard map manager database
if (!SqlDatabaseUtils.DatabaseExists(Configuration.ShardMapManagerServerName, Configuration.ShardMapManagerDatabaseName))
{
SqlDatabaseUtils.CreateDatabase(Configuration.ShardMapManagerServerName, Configuration.ShardMapManagerDatabaseName);
}
// Create shard map manager
string shardMapManagerConnectionString =
Configuration.GetConnectionString(
Configuration.ShardMapManagerServerName,
Configuration.ShardMapManagerDatabaseName);
s_shardMapManager = ShardManagementUtils.CreateOrGetShardMapManager(shardMapManagerConnectionString);
// Create shard map
ListShardMap <int> shardMap = ShardManagementUtils.CreateOrGetRangeShardMap <int>(
s_shardMapManager, Configuration.ShardMapName);
// Create schema info so that the split-merge service can be used to move data in sharded tables
// and reference tables.
CreateSchemaInfo(shardMap.Name);
// If there are no shards, add two shards: one for [0,100) and one for [100,+inf)
if (!shardMap.GetShards().Any())
{
CreateShardSample.CreateShard(shardMap, 1);
CreateShardSample.CreateShard(shardMap, 2);
}
}
public List <string[]> ExecuteMultiShardQuery(ListShardMap <int> shardMap, string credentialsConnectionString)
{
// Get the shards to connect to
IEnumerable <Shard> shards = shardMap.GetShards();
// Create the multi-shard connection
using (MultiShardConnection conn = new MultiShardConnection(shards, credentialsConnectionString))
{
// Create a simple command
using (MultiShardCommand cmd = conn.CreateCommand())
{
// Because this query is grouped by CustomerID, which is sharded,
// we will not get duplicate rows.
cmd.CommandText = @"
SELECT
c.CustomerId,
c.Name AS CustomerName,
o.OrderID AS OrderCount
FROM
dbo.Customers AS c INNER JOIN
dbo.Orders AS o
ON c.CustomerID = o.CustomerID
ORDER BY
1";
// Append a column with the shard name where the row came from
cmd.ExecutionOptions = MultiShardExecutionOptions.IncludeShardNameColumn;
// Allow for partial results in case some shards do not respond in time
cmd.ExecutionPolicy = MultiShardExecutionPolicy.PartialResults;
// Allow the entire command to take up to 30 seconds
cmd.CommandTimeout = 30;
// Execute the command.
// We do not need to specify retry logic because MultiShardDataReader will internally retry until the CommandTimeout expires.
using (MultiShardDataReader reader = cmd.ExecuteReader())
{
// Get the column names
TableFormatter formatter = new TableFormatter(GetColumnNames(reader).ToArray());
int rows = 0;
while (reader.Read())
{
// Read the values using standard DbDataReader methods
object[] values = new object[reader.FieldCount];
reader.GetValues(values);
// Extract just database name from the $ShardLocation pseudocolumn to make the output formater cleaner.
// Note that the $ShardLocation pseudocolumn is always the last column
int shardLocationOrdinal = values.Length - 1;
values[shardLocationOrdinal] = ExtractDatabaseName(values[shardLocationOrdinal].ToString());
// Add values to output formatter
formatter.AddRow(values);
rows++;
}
Console.WriteLine(formatter.ToString());
Console.WriteLine("({0} rows returned)", rows);
return(formatter._rows);
}
}
}
}
public void ConcurrencyScenarioListShardMap()
{
bool operationFailed; // variable to track status of negative test scenarios
// Create 2 SMM objects representing management and client
ShardMapManager smmMgmt = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
ShardMapManager smmClient = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
#region CreateShardMap
// Add a shard map from management SMM.
ShardMap smMgmt = smmMgmt.CreateListShardMap <int>(ShardMapManagerConcurrencyTests.s_shardMapName);
Assert.AreEqual(ShardMapManagerConcurrencyTests.s_shardMapName, smMgmt.Name);
// Lookup shard map from client SMM.
ShardMap smClient = smmClient.GetShardMap(ShardMapManagerConcurrencyTests.s_shardMapName);
Assert.IsNotNull(smClient);
#endregion CreateShardMap
#region ConvertToListShardMap
ListShardMap <int> lsmMgmt = smmMgmt.GetListShardMap <int>(ShardMapManagerConcurrencyTests.s_shardMapName);
Assert.IsNotNull(lsmMgmt);
// look up shard map again, it will
ListShardMap <int> lsmClient = smmClient.GetListShardMap <int>(ShardMapManagerConcurrencyTests.s_shardMapName);
Assert.IsNotNull(lsmClient);
#endregion ConvertToListShardMap
#region DeleteShardMap
// verify that smClient is accessible
IEnumerable <Shard> shardClient = lsmClient.GetShards();
smmMgmt.DeleteShardMap(lsmMgmt);
operationFailed = false;
try
{
// smClient does not exist, below call will fail.
IEnumerable <Shard> sCNew = lsmClient.GetShards();
}
catch (ShardManagementException sme)
{
Assert.AreEqual(ShardManagementErrorCategory.ShardMap, sme.ErrorCategory);
Assert.AreEqual(ShardManagementErrorCode.ShardMapDoesNotExist, sme.ErrorCode);
operationFailed = true;
}
Assert.IsTrue(operationFailed);
#endregion DeleteShardMap
}