public void TestCopyLSMToGSM()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RangeShardMap<int> rsm = smm.GetRangeShardMap<int>(RecoveryManagerTests.s_rangeShardMapName);
Assert.IsNotNull(rsm);
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s = rsm.CreateShard(sl);
Assert.IsNotNull(s);
RangeMapping<int> r1 = rsm.CreateRangeMapping(new Range<int>(1, 10), s);
// Add a range to the gsm
RangeMapping<int> r2 = rsm.CreateRangeMapping(new Range<int>(11, 20), s);
Assert.IsNotNull(r1);
// Delete everything from GSM (yes, this is overkill.)
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand(String.Format("delete from {0}.__ShardManagement.ShardMappingsGlobal", Globals.ShardMapManagerDatabaseName), conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
// Briefly validate that there are, in fact, the two ranges of inconsistency we are expecting.
IEnumerable<RecoveryToken> gs = rm.DetectMappingDifferences(sl);
Assert.AreEqual(1, gs.Count(), "The test environment was not expecting more than one local shardmap.");
// Briefly validate that
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(2, kvps.Keys.Count, "The count of differences does not match the expected.");
foreach (var kvp in kvps)
{
ShardRange range = kvp.Key;
MappingLocation mappingLocation = kvp.Value;
Assert.AreEqual(MappingLocation.MappingInShardOnly, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
}
// Recover the GSM from the LSM
rm.ResolveMappingDifferences(g, MappingDifferenceResolution.KeepShardMapping);
}
// Validate that there are no more differences.
IEnumerable<RecoveryToken> gsAfterFix = rm.DetectMappingDifferences(sl);
foreach (RecoveryToken g in gsAfterFix)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Keys.Count, "There were still differences after resolution.");
}
}
public void TestGeoFailoverAttach()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
ListShardMap<int> listsm = smm.GetListShardMap<int>(RecoveryManagerTests.s_listShardMapName);
Assert.IsNotNull(listsm);
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
ShardLocation slNew = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0] + "_new");
// deploy LSM version 1.1 at location 'sl' before calling CreateShard() so that createshard will not deploy latest LSM version
smm.UpgradeLocalStore(sl, new Version(1, 1));
Shard s = listsm.CreateShard(sl);
Assert.IsNotNull(s);
for (int i = 0; i < 5; i++)
{
PointMapping<int> r = listsm.CreatePointMapping(creationInfo: new PointMappingCreationInfo<int>(i, s, MappingStatus.Online));
Assert.IsNotNull(r);
}
// rename shard1 as shard1_new
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand("alter database shard1 set single_user with rollback immediate", conn))
{
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = new SqlCommand("alter database shard1 modify name = shard1_new", conn))
{
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = new SqlCommand("alter database shard1_new set multi_user with rollback immediate", conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
rm.DetachShard(sl);
rm.AttachShard(slNew);
// Verify that shard location in LSM is updated to show databasename as 'shard1_new'
IStoreResults result;
using (IStoreOperationLocal op = smm.StoreOperationFactory.CreateGetShardsLocalOperation(
smm,
slNew,
"RecoveryTest"))
{
result = op.Do();
}
Assert.AreEqual("shard1_new", result.StoreShards.First().Location.Database);
// detect mapping differences and add local mappings to GSM
var gs = rm.DetectMappingDifferences(slNew);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(5, kvps.Keys.Count, "Count of Mapping differences for shard1_new does not match expected value.");
rm.ResolveMappingDifferences(g, MappingDifferenceResolution.KeepShardMapping);
}
gs = rm.DetectMappingDifferences(slNew);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Keys.Count, "GSM and LSM at shard1_new do not have consistent mappings");
}
// rename shard1_new back to shard1 so that test cleanup operations will succeed
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand("alter database shard1_new set single_user with rollback immediate", conn))
{
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = new SqlCommand("alter database shard1_new modify name = shard1", conn))
{
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = new SqlCommand("alter database shard1 set multi_user with rollback immediate", conn))
{
cmd.ExecuteNonQuery();
}
}
}
public void TestPointMappingRecoverFromLSM()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
ListShardMap<int> listsm = smm.GetListShardMap<int>(RecoveryManagerTests.s_listShardMapName);
Assert.IsNotNull(listsm);
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s = listsm.CreateShard(sl);
Assert.IsNotNull(s);
for (int i = 0; i < 5; i++)
{
PointMapping<int> r = listsm.CreatePointMapping(creationInfo: new PointMappingCreationInfo<int>(i, s, MappingStatus.Online));
Assert.IsNotNull(r);
}
// Delete all the ranges and shardmaps from the shardlocation.
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand("delete from shard1.__ShardManagement.ShardMappingsLocal", conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
var gs = rm.DetectMappingDifferences(sl);
// Validate that all the shard locations are in fact missing from the LSM.
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(5, kvps.Keys.Count, "The count of differences does not match the expected.");
foreach (var kvp in kvps)
{
ShardRange range = kvp.Key;
MappingLocation mappingLocation = kvp.Value;
Assert.AreEqual(MappingLocation.MappingInShardMapOnly, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
}
rm.RebuildMappingsOnShard(g, kvps.Keys.Take(3));
}
gs = rm.DetectMappingDifferences(sl);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(2, kvps.Values.Where(loc => loc != MappingLocation.MappingInShardMapAndShard).Count(), "The count of differences does not match the expected.");
// We expect that the last two ranges only are missing from the shards.
var expectedLocations = new List<MappingLocation>()
{
MappingLocation.MappingInShardMapAndShard,
MappingLocation.MappingInShardMapAndShard,
MappingLocation.MappingInShardMapAndShard,
MappingLocation.MappingInShardMapOnly,
MappingLocation.MappingInShardMapOnly,
};
Assert.IsTrue(expectedLocations.Zip(kvps.Values, (x, y) => x == y).Aggregate((x, y) => x && y), "RebuildRangeShardMap rebuilt the shards out of order with respect to its keeplist.");
// Rebuild the range, leaving 1 inconsistency
rm.RebuildMappingsOnShard(g, kvps.Keys.Skip(1));
}
gs = rm.DetectMappingDifferences(sl);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(1, kvps.Values.Where(loc => loc != MappingLocation.MappingInShardMapAndShard).Count(), "The count of differences does not match the expected.");
// Rebuild the range, leaving no inconsistencies
rm.RebuildMappingsOnShard(g, kvps.Keys);
}
gs = rm.DetectMappingDifferences(sl);
// Everything should be semantically consistent now.
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Values.Where(loc => loc != MappingLocation.MappingInShardMapAndShard).Count(), "The count of differences does not match the expected.");
// As a sanity check, make sure the root is restorable from this LSM.
rm.ResolveMappingDifferences(g, MappingDifferenceResolution.KeepShardMapping);
}
gs = rm.DetectMappingDifferences(sl);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Keys.Count, "The GSM is not restorable from a rebuilt local shard.");
}
}
public void TestPointMappingRecoverFromGSM()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
ListShardMap<int> listsm = smm.GetListShardMap<int>(RecoveryManagerTests.s_listShardMapName);
Assert.IsNotNull(listsm);
ShardLocation sl = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s = listsm.CreateShard(sl);
Assert.IsNotNull(s);
for (int i = 0; i < 5; i++)
{
PointMapping<int> r = listsm.CreatePointMapping(creationInfo: new PointMappingCreationInfo<int>(i, s, MappingStatus.Online));
Assert.IsNotNull(r);
}
// Delete all the ranges and shardmaps from the shardlocation.
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand("delete from shard1.__ShardManagement.ShardMappingsLocal", conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
var gs = rm.DetectMappingDifferences(sl);
// Validate that all the shard locations are in fact missing from the LSM.
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(5, kvps.Keys.Count, "The count of differences does not match the expected.");
foreach (var kvp in kvps)
{
ShardRange range = kvp.Key;
MappingLocation mappingLocation = kvp.Value;
Assert.AreEqual(MappingLocation.MappingInShardMapOnly, mappingLocation, "An unexpected difference between global and local shardmaps was detected. This is likely a false positive and implies a bug in the detection code.");
}
// Recover the LSM from the GSM
rm.ResolveMappingDifferences(g, MappingDifferenceResolution.KeepShardMapMapping);
}
gs = rm.DetectMappingDifferences(sl);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(0, kvps.Values.Count(), "The count of differences does not match the expected.");
}
}
public void TestRebuildShardFromGSMRangeKeepNonconflicts()
{
ShardMapManager smm = ShardMapManagerFactory.GetSqlShardMapManager(
Globals.ShardMapManagerConnectionString,
ShardMapManagerLoadPolicy.Lazy);
RangeShardMap<int> rsm = smm.GetRangeShardMap<int>(RecoveryManagerTests.s_rangeShardMapName);
Assert.IsNotNull(rsm);
ShardLocation sl1 = new ShardLocation(Globals.ShardMapManagerTestsDatasourceName, RecoveryManagerTests.s_shardedDBs[0]);
Shard s1 = rsm.CreateShard(sl1);
// set initial ranges as non-intersecting.
RangeMapping<int> r1 = rsm.CreateRangeMapping(new Range<int>(1, 6), s1);
RangeMapping<int> r2 = rsm.CreateRangeMapping(new Range<int>(6, 10), s1);
// Only mess up the range on the right.
using (SqlConnection conn = new SqlConnection(Globals.ShardMapManagerTestConnectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand("update shard1.__ShardManagement.ShardMappingsLocal set MaxValue = 0x8000000B, MappingId = newid() where MaxValue = 0x8000000A", conn))
{
cmd.ExecuteNonQuery();
}
using (SqlCommand cmd = new SqlCommand("update shard1.__ShardManagement.ShardMappingsLocal set MinValue = 0x8000000B where MinValue = 0x8000000A", conn))
{
cmd.ExecuteNonQuery();
}
}
RecoveryManager rm = new RecoveryManager(smm);
var gs = rm.DetectMappingDifferences(sl1);
foreach (RecoveryToken g in gs)
{
var kvps = rm.GetMappingDifferences(g);
Assert.AreEqual(2, kvps.Keys.Count, "The count of differences does not match the expected.");
// Let's make sure that rebuild does not unintuitively delete ranges 1-6.
rm.RebuildMappingsOnShard(g, new List<ShardRange>());
}
gs = rm.DetectMappingDifferences(sl1);
foreach (RecoveryToken g in gs)
{
// Take local.
rm.ResolveMappingDifferences(g, MappingDifferenceResolution.KeepShardMapping);
}
var resultingMappings = rsm.GetMappings(new Range<int>(1, 11), s1);
// Make sure the mapping [1-6) is still around.
Assert.AreEqual(1, resultingMappings.Count(), "RebuildShard unexpectedly removed a non-conflicting range.");
}