public void TestRestoreGSMFromLSMsRangeWithGarbageInGSM()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RangeShardMap<int> rsm = smm.GetRangeShardMap<int>(RecoveryManagerTests.s_rangeShardMapName);
Assert.IsNotNull(rsm);
IList<ShardLocation> sls = new List<ShardLocation>();
int i = 0;
List<RangeMapping<int>> ranges = new List<RangeMapping<int>>();
foreach (string dbName in RecoveryManagerTests.s_shardedDBs)
{
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, dbName);
sls.Add(sl);
Shard s = rsm.CreateShard(sl);
Assert.IsNotNull(s);
var r = rsm.CreateRangeMapping(new Range<int>(1 + i * 10, 10 + i * 10), s);
Assert.IsNotNull(r);
ranges.Add(r);
i++;
}
// Perturb the mappings in the GSM.
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand(String.Format("update {0}.__ShardManagement.ShardMappingsGlobal set MaxValue = MaxValue + 1, MinValue = MinValue + 1", Globals.ShardMapManagerDatabaseName), conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
// Validate that we detect the inconsistencies in all the LSMs.
foreach (ShardLocation sl in sls)
{
IEnumerable<RecoveryToken> gs = rm.DetectMappingDifferences(sl);
Assert.AreEqual(1, gs.Count(), "The test environment was not expecting more than one local shardmap.");
// Briefly validate that
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(2, kvps.Keys.Count, "The count of differences does not match the expected.");
}
}
// Recover the LSM from the GSM
rm.RebuildMappingsOnShardMapManagerFromShards(sls);
// Validate that we fixed all the inconsistencies.
foreach (ShardLocation sl in sls)
{
IEnumerable<RecoveryToken> gs = rm.DetectMappingDifferences(sl);
// Briefly validate that
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Keys.Count, "There were still differences after resolution.");
}
}
}
public static void ShardMapManagerLoadTestsCleanup()
{
// Clear all connection pools.
SqlConnection.ClearAllPools();
// Detect inconsistencies for all shard locations in a shard map.
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RecoveryManager rm = new RecoveryManager(smm);
bool inconsistencyDetected = false;
foreach (ShardLocation sl in smm.GetDistinctShardLocations())
{
IEnumerable<RecoveryToken> gs = rm.DetectMappingDifferences(sl);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
if (kvps.Keys.Count > 0)
{
inconsistencyDetected = true;
Debug.WriteLine("LSM at location {0} is not consistent with GSM", sl);
}
}
}
bool deadlocksDetected = false;
// Check for deadlocks during the run and cleanup database and deadlock objects on successful run
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
// check for any deadlocks occured during the run and cleanup deadlock monitoring objects
using (SqlCommand cmd = conn.CreateCommand())
{
cmd.CommandText = s_deadlockDetectionQuery;
cmd.CommandType = System.Data.CommandType.Text;
try
{
using (SqlDataReader reader = cmd.ExecuteReader())
{
if (reader.HasRows)
{
// some deadlocks occured during the test, collect xml plan for these deadlocks
deadlocksDetected = true;
while (reader.Read())
{
Debug.WriteLine("Deadlock information");
Debug.WriteLine(reader.GetSqlXml(0).Value);
}
}
}
}
catch (SqlException)
{
}
}
// cleanup only if there are no inconsistencies and deadlocks during the run.
if (!deadlocksDetected && !inconsistencyDetected)
{
foreach (string q in s_deadlockDetectionCleanupQueries)
{
using (SqlCommand cmd = new SqlCommand(q, conn))
{
try
{
cmd.ExecuteNonQuery();
}
catch (SqlException)
{
}
}
}
// Drop shard databases
for (int i = 0; i < ShardMapManagerLoadTests.s_shardedDBs.Length; i++)
{
using (SqlCommand cmd = new SqlCommand(
string.Format(Globals.DropDatabaseQuery, ShardMapManagerLoadTests.s_shardedDBs[i]),
conn))
{
cmd.ExecuteNonQuery();
}
}
// Drop shard map manager database
using (SqlCommand cmd = new SqlCommand(
string.Format(Globals.DropDatabaseQuery, Globals.ShardMapManagerDatabaseName),
conn))
{
cmd.ExecuteNonQuery();
}
}
}
}
public void TestCopyLSMToGSM()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RangeShardMap<int> rsm = smm.GetRangeShardMap<int>(RecoveryManagerTests.s_rangeShardMapName);
Assert.IsNotNull(rsm);
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s = rsm.CreateShard(sl);
Assert.IsNotNull(s);
RangeMapping<int> r1 = rsm.CreateRangeMapping(new Range<int>(1, 10), s);
// Add a range to the gsm
RangeMapping<int> r2 = rsm.CreateRangeMapping(new Range<int>(11, 20), s);
Assert.IsNotNull(r1);
// Delete everything from GSM (yes, this is overkill.)
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand(String.Format("delete from {0}.__ShardManagement.ShardMappingsGlobal", Globals.ShardMapManagerDatabaseName), conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
// Briefly validate that there are, in fact, the two ranges of inconsistency we are expecting.
IEnumerable<RecoveryToken> gs = rm.DetectMappingDifferences(sl);
Assert.AreEqual(1, gs.Count(), "The test environment was not expecting more than one local shardmap.");
// Briefly validate that
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(2, kvps.Keys.Count, "The count of differences does not match the expected.");
foreach (var kvp in kvps)
{
ShardRange range = kvp.Key;
MappingLocation mappingLocation = kvp.Value;
Assert.AreEqual(MappingLocation.MappingInShardOnly, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
}
// Recover the GSM from the LSM
rm.ResolveMappingDifferences(g, MappingDifferenceResolution.KeepShardMapping);
}
// Validate that there are no more differences.
IEnumerable<RecoveryToken> gsAfterFix = rm.DetectMappingDifferences(sl);
foreach (RecoveryToken g in gsAfterFix)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Keys.Count, "There were still differences after resolution.");
}
}
public void TestRestoreGSMFromLSMsRange()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RangeShardMap<int> rsm = smm.GetRangeShardMap<int>(RecoveryManagerTests.s_rangeShardMapName);
Assert.IsNotNull(rsm);
IList<ShardLocation> sls = new List<ShardLocation>();
int i = 0;
List<RangeMapping<int>> ranges = new List<RangeMapping<int>>();
foreach (string dbName in RecoveryManagerTests.s_shardedDBs)
{
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, dbName);
sls.Add(sl);
Shard s = rsm.CreateShard(sl);
Assert.IsNotNull(s);
var r = rsm.CreateRangeMapping(new Range<int>(1 + i * 10, 10 + i * 10), s);
Assert.IsNotNull(r);
ranges.Add(r);
i++;
}
// Delete all mappings from GSM
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand(String.Format("delete from {0}.__ShardManagement.ShardMappingsGlobal", Globals.ShardMapManagerDatabaseName), conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
// Validate that we detect the inconsistencies in all the LSMs.
foreach (ShardLocation sl in sls)
{
IEnumerable<RecoveryToken> gs = rm.DetectMappingDifferences(sl);
Assert.AreEqual(1, gs.Count(), "The test environment was not expecting more than one local shardmap.");
// Briefly validate that
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(1, kvps.Keys.Count, "The count of differences does not match the expected.");
foreach (var kvp in kvps)
{
ShardRange range = kvp.Key;
MappingLocation mappingLocation = kvp.Value;
Assert.AreEqual(MappingLocation.MappingInShardOnly, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
}
}
}
// Recover the LSM from the GSM
rm.RebuildMappingsOnShardMapManagerFromShards(sls);
// Validate that we fixed all the inconsistencies.
foreach (ShardLocation sl in sls)
{
IEnumerable<RecoveryToken> gs = rm.DetectMappingDifferences(sl);
// Briefly validate that
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Keys.Count, "There were still differences after resolution.");
}
}
}
public void TestConsistencyDetectionAndViewingWithDisjointRanges()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RangeShardMap<int> rsm = smm.GetRangeShardMap<int>(RecoveryManagerTests.s_rangeShardMapName);
Assert.IsNotNull(rsm);
// Make sure no other rangemappings are floating around here.
var rangeMappings = rsm.GetMappings();
foreach (var rangeMapping in rangeMappings)
{
rsm.DeleteMapping(rangeMapping);
}
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s = rsm.CreateShard(sl);
Assert.IsNotNull(s);
RangeMapping<int> r1 = rsm.CreateRangeMapping(new Range<int>(1, 10), s);
// Add a range to the gsm
RangeMapping<int> r2 = rsm.CreateRangeMapping(new Range<int>(11, 20), s);
Assert.IsNotNull(r1);
// Delete the original range from the GSM.
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand(String.Format("delete from {0}.__ShardManagement.ShardMappingsGlobal where MinValue = 0x80000001", Globals.ShardMapManagerDatabaseName), conn))
{
cmd.ExecuteNonQuery();
}
}
// Delete the new range from the LSM, so the LSM and GSM now have non-intersecting ranges.
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand(String.Format("delete from shard1.__ShardManagement.ShardMappingsLocal where MinValue = 0x8000000B", Globals.ShardMapManagerDatabaseName), conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
IEnumerable<RecoveryToken> gs = rm.DetectMappingDifferences(sl);
Assert.AreEqual(1, gs.Count(), "The test environment was not expecting more than one local shardmap.");
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(2, kvps.Keys.Count, "The count of differences does not match the expected.");
foreach (var kvp in kvps)
{
ShardRange range = kvp.Key;
MappingLocation mappingLocation = kvp.Value;
if ((int)range.High.Value == 10)
{
Assert.AreEqual(MappingLocation.MappingInShardOnly, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
continue;
}
else if ((int)range.High.Value == 20)
{
Assert.AreEqual(MappingLocation.MappingInShardMapOnly, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
continue;
}
Assert.Fail("Unexpected range detected.");
}
}
}
public void TestConsistencyDetectionWithDivergence()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RangeShardMap<int> rsm = smm.GetRangeShardMap<int>(RecoveryManagerTests.s_rangeShardMapName);
Assert.IsNotNull(rsm);
ShardLocation sl1 = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
ShardLocation sl2 = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[1]);
Shard s1 = rsm.CreateShard(sl1);
Shard s2 = rsm.CreateShard(sl2);
// set initial ranges as non-intersecting.
RangeMapping<int> r1 = rsm.CreateRangeMapping(new Range<int>(1, 6), s1);
RangeMapping<int> r2 = rsm.CreateRangeMapping(new Range<int>(6, 10), s2);
// Perturb the first LSM so that it has a
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand("update shard1.__ShardManagement.ShardMappingsLocal set MaxValue = 0x8000000B", conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
var gs = rm.DetectMappingDifferences(sl1);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(2, kvps.Keys.Count, "The count of differences does not match the expected.");
// We expect 6-10, and 10-11. If we did not detect intersected ranges, and only used tagged ranges, we would have only 6-11 as a single range, which would be insufficient for rebuild.
foreach (var kvp in kvps)
{
ShardRange range = kvp.Key;
MappingLocation mappingLocation = kvp.Value;
if ((int)range.High.Value == 10)
{
Assert.AreEqual(MappingLocation.MappingInShardMapAndShard, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
continue;
}
else if ((int)range.High.Value == 11)
{
Assert.AreEqual(MappingLocation.MappingInShardOnly, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
continue;
}
Assert.Fail("Unexpected range detected.");
}
}
}
public void TestConsistencyDetectionAndViewingWithWiderRangeInGSM()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RangeShardMap<int> rsm = smm.GetRangeShardMap<int>(RecoveryManagerTests.s_rangeShardMapName);
Assert.IsNotNull(rsm);
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s = rsm.CreateShard(sl);
Assert.IsNotNull(s);
RangeMapping<int> r1 = rsm.CreateRangeMapping(new Range<int>(1, 10), s);
Assert.IsNotNull(r1);
// Corrupt the gsm by increasing the max range and decreasing min range. We should see two ranges show up in the list of differences. The shared range
// in the middle artificially has the same version number, so it should not register as a conflicting range.
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand(String.Format("update {0}.__ShardManagement.ShardMappingsGlobal set MinValue = MinValue - 1, MaxValue = MaxValue + 1", Globals.ShardMapManagerDatabaseName), conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
IEnumerable<RecoveryToken> gs = rm.DetectMappingDifferences(sl);
Assert.AreEqual(1, gs.Count(), "The test environment was not expecting more than one local shardmap.");
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(2, kvps.Keys.Count, "The count of differences does not match the expected.");
foreach (var kvp in kvps)
{
ShardRange range = kvp.Key;
MappingLocation mappingLocation = kvp.Value;
Assert.AreEqual(1, (int)range.High.Value - (int)range.Low.Value, "The ranges reported differed from those expected.");
Assert.AreEqual(MappingLocation.MappingInShardMapOnly, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
}
}
}
public void TestConsistencyDetectionOnListMapping()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
ListShardMap<int> rsm = smm.GetListShardMap<int>(RecoveryManagerTests.s_listShardMapName);
Assert.IsNotNull(rsm);
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s = rsm.CreateShard(sl);
Assert.IsNotNull(s);
for (int i = 0; i < 5; i++)
{
PointMapping<int> p = rsm.CreatePointMapping(2 * i, s);
Assert.IsNotNull(p);
}
// Now, delete some points from both, and change the version of a shared shard mapping in the middle.
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand(String.Format("delete from {0}.__ShardManagement.ShardMappingsGlobal where MinValue IN (0x80000000, 0x80000002)", Globals.ShardMapManagerDatabaseName), conn))
{
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = new SqlCommand("delete from shard1.__ShardManagement.ShardMappingsLocal where MinValue = 0x80000008", conn))
{
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = new SqlCommand("update shard1.__ShardManagement.ShardMappingsLocal set MappingId = newid() where MinValue = 0x80000006", conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
IEnumerable<RecoveryToken> gs = rm.DetectMappingDifferences(sl);
Assert.AreEqual(1, gs.Count(), "The test environment was not expecting more than one local shardmap.");
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(4, kvps.Keys.Count, "The count of differences does not match the expected.");
Assert.AreEqual(1, kvps.Values.Where(l => l == MappingLocation.MappingInShardMapOnly).Count(), "The count of shardmap only differences does not match the expected.");
Assert.AreEqual(2, kvps.Values.Where(l => l == MappingLocation.MappingInShardOnly).Count(), "The count of shard only differences does not match the expected.");
Assert.AreEqual(1, kvps.Values.Where(l => l == MappingLocation.MappingInShardMapAndShard).Count(), "The count of shard only differences does not match the expected.");
}
}
public void TestConsistencyDetectionAndViewingWithNoConflicts()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RangeShardMap<int> rsm = smm.GetRangeShardMap<int>(RecoveryManagerTests.s_rangeShardMapName);
Assert.IsNotNull(rsm);
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s = rsm.CreateShard(sl);
Assert.IsNotNull(s);
RangeMapping<int> r1 = rsm.CreateRangeMapping(new Range<int>(1, 10), s);
Assert.IsNotNull(r1);
RecoveryManager rm = new RecoveryManager(smm);
IEnumerable<RecoveryToken> gs = rm.DetectMappingDifferences(sl);
Assert.AreEqual(1, gs.Count(), "The test environment was not expecting more than one local shardmap.");
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Keys.Count, "An unexpected conflict was detected");
foreach (var kvp in kvps)
{
ShardRange range = kvp.Key;
MappingLocation mappingLocation = kvp.Value;
Assert.AreEqual(MappingLocation.MappingInShardMapAndShard, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
}
}
}
public void TestConsistencyDetectionAndViewingWithVersionOnlyConflict()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RangeShardMap<int> rsm = smm.GetRangeShardMap<int>(RecoveryManagerTests.s_rangeShardMapName);
Assert.IsNotNull(rsm);
// Make sure no other rangemappings are floating around here.
var rangeMappings = rsm.GetMappings();
foreach (var rangeMapping in rangeMappings)
{
rsm.DeleteMapping(rangeMapping);
}
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s = rsm.CreateShard(sl);
Assert.IsNotNull(s);
RangeMapping<int> r1 = rsm.CreateRangeMapping(new Range<int>(1, 10), s);
Assert.IsNotNull(r1);
// Corrupt the mapping id number on the global shardmap.
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand(String.Format("update {0}.__ShardManagement.ShardMappingsGlobal set MappingId = newid()", Globals.ShardMapManagerDatabaseName), conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
IEnumerable<RecoveryToken> gs = rm.DetectMappingDifferences(sl);
Assert.AreEqual(1, gs.Count(), "The test environment was not expecting more than one local shardmap.");
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(1, kvps.Keys.Count, "An unexpected conflict was detected");
foreach (var kvp in kvps)
{
ShardRange range = kvp.Key;
MappingLocation mappingLocation = kvp.Value;
Assert.AreEqual(MappingLocation.MappingInShardMapAndShard, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
}
}
}
public void TestDetachAttachShard()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RangeShardMap<int> rsm = smm.GetRangeShardMap<int>(RecoveryManagerTests.s_rangeShardMapName);
Assert.IsNotNull(rsm);
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s = rsm.CreateShard(sl);
Assert.IsNotNull(s);
// I am still not fully clear on the use case for AttachShard and DetachShard, but here's a simple test validating that
// they don't throw exceptions if they get called against themselves.
RecoveryManager rm = new RecoveryManager(smm);
rm.DetachShard(sl);
rm.AttachShard(sl);
}
public void TestGeoFailoverAttach()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
ListShardMap<int> listsm = smm.GetListShardMap<int>(RecoveryManagerTests.s_listShardMapName);
Assert.IsNotNull(listsm);
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
ShardLocation slNew = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0] + "_new");
// deploy LSM version 1.1 at location 'sl' before calling CreateShard() so that createshard will not deploy latest LSM version
smm.UpgradeLocalStore(sl, new Version(1, 1));
Shard s = listsm.CreateShard(sl);
Assert.IsNotNull(s);
for (int i = 0; i < 5; i++)
{
PointMapping<int> r = listsm.CreatePointMapping(creationInfo: new PointMappingCreationInfo<int>(i, s, MappingStatus.Online));
Assert.IsNotNull(r);
}
// rename shard1 as shard1_new
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand("alter database shard1 set single_user with rollback immediate", conn))
{
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = new SqlCommand("alter database shard1 modify name = shard1_new", conn))
{
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = new SqlCommand("alter database shard1_new set multi_user with rollback immediate", conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
rm.DetachShard(sl);
rm.AttachShard(slNew);
// Verify that shard location in LSM is updated to show databasename as 'shard1_new'
IStoreResults result;
using (IStoreOperationLocal op = smm.StoreOperationFactory.CreateGetShardsLocalOperation(
smm,
slNew,
"RecoveryTest"))
{
result = op.Do();
}
Assert.AreEqual("shard1_new", result.StoreShards.First().Location.Database);
// detect mapping differences and add local mappings to GSM
var gs = rm.DetectMappingDifferences(slNew);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(5, kvps.Keys.Count, "Count of Mapping differences for shard1_new does not match expected value.");
rm.ResolveMappingDifferences(g, MappingDifferenceResolution.KeepShardMapping);
}
gs = rm.DetectMappingDifferences(slNew);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Keys.Count, "GSM and LSM at shard1_new do not have consistent mappings");
}
// rename shard1_new back to shard1 so that test cleanup operations will succeed
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand("alter database shard1_new set single_user with rollback immediate", conn))
{
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = new SqlCommand("alter database shard1_new modify name = shard1", conn))
{
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = new SqlCommand("alter database shard1 set multi_user with rollback immediate", conn))
{
cmd.ExecuteNonQuery();
}
}
}
public void TestPointMappingRecoverFromGSM()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
ListShardMap<int> listsm = smm.GetListShardMap<int>(RecoveryManagerTests.s_listShardMapName);
Assert.IsNotNull(listsm);
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s = listsm.CreateShard(sl);
Assert.IsNotNull(s);
for (int i = 0; i < 5; i++)
{
PointMapping<int> r = listsm.CreatePointMapping(creationInfo: new PointMappingCreationInfo<int>(i, s, MappingStatus.Online));
Assert.IsNotNull(r);
}
// Delete all the ranges and shardmaps from the shardlocation.
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand("delete from shard1.__ShardManagement.ShardMappingsLocal", conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
var gs = rm.DetectMappingDifferences(sl);
// Validate that all the shard locations are in fact missing from the LSM.
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(5, kvps.Keys.Count, "The count of differences does not match the expected.");
foreach (var kvp in kvps)
{
ShardRange range = kvp.Key;
MappingLocation mappingLocation = kvp.Value;
Assert.AreEqual(MappingLocation.MappingInShardMapOnly, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
}
// Recover the LSM from the GSM
rm.ResolveMappingDifferences(g, MappingDifferenceResolution.KeepShardMapMapping);
}
gs = rm.DetectMappingDifferences(sl);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Values.Count(), "The count of differences does not match the expected.");
}
}
public void TestPointMappingRecoverFromLSM()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
ListShardMap<int> listsm = smm.GetListShardMap<int>(RecoveryManagerTests.s_listShardMapName);
Assert.IsNotNull(listsm);
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s = listsm.CreateShard(sl);
Assert.IsNotNull(s);
for (int i = 0; i < 5; i++)
{
PointMapping<int> r = listsm.CreatePointMapping(creationInfo: new PointMappingCreationInfo<int>(i, s, MappingStatus.Online));
Assert.IsNotNull(r);
}
// Delete all the ranges and shardmaps from the shardlocation.
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand("delete from shard1.__ShardManagement.ShardMappingsLocal", conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
var gs = rm.DetectMappingDifferences(sl);
// Validate that all the shard locations are in fact missing from the LSM.
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(5, kvps.Keys.Count, "The count of differences does not match the expected.");
foreach (var kvp in kvps)
{
ShardRange range = kvp.Key;
MappingLocation mappingLocation = kvp.Value;
Assert.AreEqual(MappingLocation.MappingInShardMapOnly, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
}
rm.RebuildMappingsOnShard(g, kvps.Keys.Take(3));
}
gs = rm.DetectMappingDifferences(sl);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(2, kvps.Values.Where(loc => loc != MappingLocation.MappingInShardMapAndShard).Count(), "The count of differences does not match the expected.");
// We expect that the last two ranges only are missing from the shards.
var expectedLocations = new List<MappingLocation>()
{
MappingLocation.MappingInShardMapAndShard,
MappingLocation.MappingInShardMapAndShard,
MappingLocation.MappingInShardMapAndShard,
MappingLocation.MappingInShardMapOnly,
MappingLocation.MappingInShardMapOnly,
};
Assert.IsTrue(expectedLocations.Zip(kvps.Values, (x, y) => x == y).Aggregate((x, y) => x && y), "RebuildRangeShardMap rebuilt the shards out of order with respect to its keeplist.");
// Rebuild the range, leaving 1 inconsistency
rm.RebuildMappingsOnShard(g, kvps.Keys.Skip(1));
}
gs = rm.DetectMappingDifferences(sl);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(1, kvps.Values.Where(loc => loc != MappingLocation.MappingInShardMapAndShard).Count(), "The count of differences does not match the expected.");
// Rebuild the range, leaving no inconsistencies
rm.RebuildMappingsOnShard(g, kvps.Keys);
}
gs = rm.DetectMappingDifferences(sl);
// Everything should be semantically consistent now.
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Values.Where(loc => loc != MappingLocation.MappingInShardMapAndShard).Count(), "The count of differences does not match the expected.");
// As a sanity check, make sure the root is restorable from this LSM.
rm.ResolveMappingDifferences(g, MappingDifferenceResolution.KeepShardMapping);
}
gs = rm.DetectMappingDifferences(sl);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Keys.Count, "The GSM is not restorable from a rebuilt local shard.");
}
}
public void TestRebuildShardFromGSMRangeKeepNonconflicts()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RangeShardMap<int> rsm = smm.GetRangeShardMap<int>(RecoveryManagerTests.s_rangeShardMapName);
Assert.IsNotNull(rsm);
ShardLocation sl1 = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s1 = rsm.CreateShard(sl1);
// set initial ranges as non-intersecting.
RangeMapping<int> r1 = rsm.CreateRangeMapping(new Range<int>(1, 6), s1);
RangeMapping<int> r2 = rsm.CreateRangeMapping(new Range<int>(6, 10), s1);
// Only mess up the range on the right.
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand("update shard1.__ShardManagement.ShardMappingsLocal set MaxValue = 0x8000000B, MappingId = newid() where MaxValue = 0x8000000A", conn))
{
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = new SqlCommand("update shard1.__ShardManagement.ShardMappingsLocal set MinValue = 0x8000000B where MinValue = 0x8000000A", conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
var gs = rm.DetectMappingDifferences(sl1);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(2, kvps.Keys.Count, "The count of differences does not match the expected.");
// Let's make sure that rebuild does not unintuitively delete ranges 1-6.
rm.RebuildMappingsOnShard(g, new List<ShardRange>());
}
gs = rm.DetectMappingDifferences(sl1);
foreach (RecoveryToken g in gs)
{
// Take local.
rm.ResolveMappingDifferences(g, MappingDifferenceResolution.KeepShardMapping);
}
var resultingMappings = rsm.GetMappings(new Range<int>(1, 11), s1);
// Make sure the mapping [1-6) is still around.
Assert.AreEqual(1, resultingMappings.Count(), "RebuildShard unexpectedly removed a non-conflicting range.");
}
