private void TestShardKeyGeneric <TKey>(ShardKeyType keyType, TKey inputValue, Type realType) { // Excercise DetectType // ShardKey k1 = new ShardKey(inputValue); Assert.AreEqual(realType, k1.DataType); // Go to/from raw value ShardKey k2 = new ShardKey(keyType, inputValue); byte[] k2raw = k2.RawValue; ShardKey k3 = ShardKey.FromRawValue(keyType, k2raw); Assert.AreEqual(inputValue, k2.Value); Assert.AreEqual(inputValue, k3.Value); Assert.AreEqual(k2, k3); // verify comparisons Assert.AreEqual(0, k2.CompareTo(k3)); try { k3 = k2.GetNextKey(); Assert.IsTrue(k3 > k2); } catch (InvalidOperationException) { } }
public void TestShardKeyDeserializationAddTrailingZeroes() { foreach (ShardKeyInfo shardKeyInfo in ShardKeyInfo.AllTestShardKeyInfos) { Console.WriteLine(shardKeyInfo); int dataTypeLength = _shardKeyTypeLength[shardKeyInfo.KeyType]; if (shardKeyInfo.RawValue != null && shardKeyInfo.RawValue.Length != dataTypeLength) { // Add trailing zeroes byte[] originalRawValue = shardKeyInfo.RawValue; byte[] rawValueWithTrailingZeroes = new byte[dataTypeLength]; originalRawValue.CopyTo(rawValueWithTrailingZeroes, 0); ShardKey expectedDeserializedShardKey = shardKeyInfo.ShardKeyFromValue; ShardKey actualDeserializedShardKey = ShardKey.FromRawValue(shardKeyInfo.KeyType, rawValueWithTrailingZeroes); // Bug? Below fails when there are trailing zeroes even though the value is Equal //// Verify ShardKey with ShardKey.Equals //Assert.AreEqual( // expectedDeserializedShardKey, // actualDeserializedShardKey); // Bug? Below fails for Binary type if (shardKeyInfo.KeyType != ShardKeyType.Binary) { // Verify ShardKey.Value with value type-specific Equals AssertExtensions.AssertScalarOrSequenceEqual(expectedDeserializedShardKey.Value, actualDeserializedShardKey.Value, null); } } } }
/// <summary> /// Helper function to advance mapping iterators. /// </summary> /// <param name="iterator">The iterator to advance.</param> /// <param name="keyType">The data type of the map key.</param> /// <param name="nextMapping">Output value that will contain next mapping.</param> /// <param name="nextRange">Output value that will contain next range.</param> /// <param name="nextMinKey">Output value that will contain next min key.</param> private static void MoveToNextMapping( IEnumerator <IStoreMapping> iterator, ShardKeyType keyType, out IStoreMapping nextMapping, out ShardRange nextRange, out ShardKey nextMinKey) { nextMapping = iterator.MoveNext() ? iterator.Current : null; nextRange = nextMapping != null ? new ShardRange( ShardKey.FromRawValue(keyType, nextMapping.MinValue), ShardKey.FromRawValue(keyType, nextMapping.MaxValue)) : null; nextMinKey = nextRange != null ? nextRange.Low : null; }
/// <summary> /// Helper function that produces a list of MappingComparisonResults from union of points in the gsmMappings and lsmMappings. /// </summary> /// <param name="ssm">StoreShardmap to be referenced in produced MappingComparisonResults</param> /// <param name="gsmMappings">List of mappings from the GSM.</param> /// <param name="lsmMappings">List of mappings from the LSM.</param> /// <returns>List of mappingcomparisonresults: one for each range arising from the union of boundaries in gsmMappings and lsmMappings.</returns> internal static List <MappingComparisonResult> ComparePointMappings( IStoreShardMap ssm, IEnumerable <IStoreMapping> gsmMappings, IEnumerable <IStoreMapping> lsmMappings) { ShardKeyType keyType = ssm.KeyType; // Get a Linq-able set of points from the input mappings. // IDictionary <ShardKey, IStoreMapping> gsmPoints = gsmMappings.ToDictionary(gsmMapping => ShardKey.FromRawValue(keyType, gsmMapping.MinValue)); IDictionary <ShardKey, IStoreMapping> lsmPoints = lsmMappings.ToDictionary(lsmMapping => ShardKey.FromRawValue(keyType, lsmMapping.MinValue)); // Construct the output list. This is the concatenation of 3 mappings: // 1.) Intersection (the key exists in both the shardmap and the shard.) // 2.) Shard only (the key exists only in the shard.) // 3.) Shardmap only (the key exists only in the shardmap.) // List <MappingComparisonResult> results = (new List <MappingComparisonResult>()).Concat( // Intersection. lsmPoints.Keys.Intersect(gsmPoints.Keys).Select( commonPoint => new MappingComparisonResult( ssm, new ShardRange(commonPoint, commonPoint.GetNextKey()), MappingLocation.MappingInShardMapAndShard, gsmPoints[commonPoint], lsmPoints[commonPoint])) ).Concat( // Lsm only. lsmPoints.Keys.Except(gsmPoints.Keys).Select( lsmOnlyPoint => new MappingComparisonResult( ssm, new ShardRange(lsmOnlyPoint, lsmOnlyPoint.GetNextKey()), MappingLocation.MappingInShardOnly, null, lsmPoints[lsmOnlyPoint])) ).Concat( // Gsm only. gsmPoints.Keys.Except(lsmPoints.Keys).Select( gsmOnlyPoint => new MappingComparisonResult( ssm, new ShardRange(gsmOnlyPoint, gsmOnlyPoint.GetNextKey()), MappingLocation.MappingInShardMapOnly, gsmPoints[gsmOnlyPoint], null))).ToList(); return(results); }
public IEnumerable <RecoveryToken> DetectMappingDifferences(ShardLocation location, string shardMapName) { ExceptionUtils.DisallowNullArgument(location, "location"); IList <RecoveryToken> listOfTokens = new List <RecoveryToken>(); IStoreResults getShardsLocalResult; using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateGetShardsLocalOperation( this.Manager, location, "DetectMappingDifferences")) { getShardsLocalResult = op.Do(); } Debug.Assert(getShardsLocalResult.Result == StoreResult.Success); IEnumerable <IStoreShardMap> shardMaps = shardMapName == null ? getShardsLocalResult.StoreShardMaps : getShardsLocalResult.StoreShardMaps.Where(s => s.Name == shardMapName); IEnumerable <Tuple <IStoreShardMap, IStoreShard> > shardInfos = shardMaps .Select(sm => new Tuple <IStoreShardMap, IStoreShard>( sm, getShardsLocalResult.StoreShards.SingleOrDefault(s => s.ShardMapId == sm.Id))); foreach (Tuple <IStoreShardMap, IStoreShard> shardInfo in shardInfos) { IStoreShardMap ssmLocal = shardInfo.Item1; IStoreShard ssLocal = shardInfo.Item2; RecoveryToken token = new RecoveryToken(); listOfTokens.Add(token); this.StoreShardMaps[token] = shardInfo; this.Locations[token] = location; this.Inconsistencies[token] = new Dictionary <ShardRange, MappingDifference>(); DefaultStoreShard dss = new DefaultStoreShard( ssLocal.Id, ssLocal.Version, ssLocal.ShardMapId, ssLocal.Location, ssLocal.Status); // First get all local mappings. IStoreResults lsmMappings; using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeLocalOperation( this.Manager, location, "DetectMappingDifferences", ssmLocal, dss, null, true)) { lsmMappings = op.Do(); if (lsmMappings.Result == StoreResult.ShardMapDoesNotExist) { // The shard needs to be re-attached. We are ignoring these errors in // DetectMappingDifferences, since corruption is more profound than // just inconsistent mappings. // Alternatively, this shard belongs to a different shard map manager. // Either way, we can't do anything about it here. continue; } } // Next build up a set of relevant global mappings. // This is the union of those mappings that are associated with this local shard // and those mappings which intersect with mappings found in the local shard. // We will partition these mappings based on ranges. IDictionary <ShardRange, IStoreMapping> relevantGsmMappings = new Dictionary <ShardRange, IStoreMapping>(); IStoreResults gsmMappingsByMap; using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeGlobalOperation( this.Manager, "DetectMappingDifferences", ssmLocal, dss, null, ShardManagementErrorCategory.Recovery, false, true /* ignore failures */)) { gsmMappingsByMap = op.Do(); } if (gsmMappingsByMap.Result == StoreResult.ShardMapDoesNotExist) { // The shard map is not properly attached to this GSM. // This is beyond what we can handle resolving mappings. continue; } foreach (IStoreMapping gsmMapping in gsmMappingsByMap.StoreMappings) { ShardKey min = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MinValue); ShardKey max = null; switch (ssmLocal.MapType) { case ShardMapType.Range: max = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MaxValue); break; default: Debug.Assert(ssmLocal.MapType == ShardMapType.List); max = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MinValue).GetNextKey(); break; } ShardRange range = new ShardRange(min, max); relevantGsmMappings[range] = gsmMapping; } // Next, for each of the mappings in lsmMappings, we need to augment // the gsmMappingsByMap by intersecting ranges. foreach (IStoreMapping lsmMapping in lsmMappings.StoreMappings) { ShardKey min = ShardKey.FromRawValue(ssmLocal.KeyType, lsmMapping.MinValue); IStoreResults gsmMappingsByRange; if (ssmLocal.MapType == ShardMapType.Range) { ShardKey max = ShardKey.FromRawValue(ssmLocal.KeyType, lsmMapping.MaxValue); ShardRange range = new ShardRange(min, max); using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeGlobalOperation( this.Manager, "DetectMappingDifferences", ssmLocal, null, range, ShardManagementErrorCategory.Recovery, false, true /* ignore failures */)) { gsmMappingsByRange = op.Do(); } if (gsmMappingsByRange.Result == StoreResult.ShardMapDoesNotExist) { // The shard was not properly attached. // This is more than we can deal with in mapping resolution. continue; } } else { Debug.Assert(ssmLocal.MapType == ShardMapType.List); using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateFindMappingByKeyGlobalOperation( this.Manager, "DetectMappingDifferences", ssmLocal, min, CacheStoreMappingUpdatePolicy.OverwriteExisting, ShardManagementErrorCategory.Recovery, false, true /* ignore failures */)) { gsmMappingsByRange = op.Do(); if (gsmMappingsByRange.Result == StoreResult.MappingNotFoundForKey || gsmMappingsByRange.Result == StoreResult.ShardMapDoesNotExist) { // * No intersections being found is fine. Skip to the next mapping. // * The shard was not properly attached. // This is more than we can deal with in mapping resolution. continue; } } } foreach (IStoreMapping gsmMapping in gsmMappingsByRange.StoreMappings) { ShardKey retrievedMin = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MinValue); ShardRange retrievedRange = null; switch (ssmLocal.MapType) { case ShardMapType.Range: ShardKey retrievedMax = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MaxValue); retrievedRange = new ShardRange(retrievedMin, retrievedMax); break; default: Debug.Assert(ssmLocal.MapType == ShardMapType.List); retrievedMax = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MinValue).GetNextKey(); retrievedRange = new ShardRange(retrievedMin, retrievedMax); break; } relevantGsmMappings[retrievedRange] = gsmMapping; } } IList <MappingComparisonResult> comparisonResults = null; switch (ssmLocal.MapType) { case ShardMapType.Range: comparisonResults = MappingComparisonUtils.CompareRangeMappings( ssmLocal, relevantGsmMappings.Values, lsmMappings.StoreMappings); break; default: Debug.Assert(ssmLocal.MapType == ShardMapType.List); comparisonResults = MappingComparisonUtils.ComparePointMappings( ssmLocal, relevantGsmMappings.Values, lsmMappings.StoreMappings); break; } // Now we have 2 sets of mappings. Each submapping generated from this function is // 1.) in the GSM only: report. // 2.) in the LSM only: report. // 3.) in both but with different version number: report. // 4.) in both with the same version number: skip. foreach (MappingComparisonResult r in comparisonResults) { switch (r.MappingLocation) { case MappingLocation.MappingInShardMapOnly: case MappingLocation.MappingInShardOnly: break; default: Debug.Assert(r.MappingLocation == MappingLocation.MappingInShardMapAndShard); if (r.ShardMapManagerMapping.Id == r.ShardMapping.Id) { // No conflict found, skip to the next range. continue; } break; } // Store the inconsistency for later reporting. this.Inconsistencies[token][r.Range] = new MappingDifference( type: MappingDifferenceType.Range, location: r.MappingLocation, shardMap: r.ShardMap, mappingForShard: r.ShardMapping, mappingForShardMap: r.ShardMapManagerMapping); } } return(listOfTokens); }
public void ShardKeyTests() { ShardKey key = null; ShardKey result = null; byte[] array = null; byte[] arraymax = null; // Verify boundary conditions result = maxNonNullKey32.GetNextKey(); Debug.Assert(result.IsMax); Debug.Assert(result == ShardKey.MaxInt32); result = maxNonNullKey64.GetNextKey(); Debug.Assert(result.IsMax); Debug.Assert(result == ShardKey.MaxInt64); array = Enumerable.Repeat((byte)0xff, 16).ToArray(); key = ShardKey.FromRawValue(ShardKeyType.Guid, array); // can not use other ctor because normalized representation differ result = key.GetNextKey(); Debug.Assert(result.IsMax); Debug.Assert(result == ShardKey.MaxGuid); array = Enumerable.Repeat((byte)0xff, 128).ToArray(); key = new ShardKey(array); result = key.GetNextKey(); Debug.Assert(result.IsMax); Debug.Assert(result == ShardKey.MaxBinary); key = new ShardKey(max32 - 1); result = key.GetNextKey(); Debug.Assert(result == maxNonNullKey32); key = new ShardKey(max64 - 1); result = key.GetNextKey(); Debug.Assert(result == maxNonNullKey64); arraymax = Enumerable.Repeat((byte)0xff, 16).ToArray(); array = Enumerable.Repeat((byte)0xff, 16).ToArray(); array[15] = 0xfe; key = ShardKey.FromRawValue(ShardKeyType.Guid, array); // can not use other ctor because normalized representation differ result = key.GetNextKey(); Debug.Assert(result == ShardKey.FromRawValue(ShardKeyType.Guid, arraymax)); arraymax = Enumerable.Repeat((byte)0xff, 128).ToArray(); array = Enumerable.Repeat((byte)0xff, 128).ToArray(); array[127] = 0xfe; key = new ShardKey(array); result = key.GetNextKey(); Debug.Assert(result == ShardKey.FromRawValue(ShardKeyType.Binary, arraymax)); key = new ShardKey(ShardKeyType.Int32, null); AssertExtensions.AssertThrows <InvalidOperationException>(() => key.GetNextKey()); key = new ShardKey(ShardKeyType.Int64, null); AssertExtensions.AssertThrows <InvalidOperationException>(() => key.GetNextKey()); key = new ShardKey(ShardKeyType.Guid, null); AssertExtensions.AssertThrows <InvalidOperationException>(() => key.GetNextKey()); key = new ShardKey(ShardKeyType.Binary, null); AssertExtensions.AssertThrows <InvalidOperationException>(() => key.GetNextKey()); result = ShardKey.MinInt32.GetNextKey(); Debug.Assert(result == new ShardKey(Int32.MinValue + 1)); result = ShardKey.MinInt64.GetNextKey(); Debug.Assert(result == new ShardKey(Int64.MinValue + 1)); result = ShardKey.MinGuid.GetNextKey(); array = new byte[16]; array[15] = 0x01; key = ShardKey.FromRawValue(ShardKeyType.Guid, array); Debug.Assert(result == key); result = ShardKey.MinBinary.GetNextKey(); array = new byte[128]; array[127] = 0x01; key = ShardKey.FromRawValue(ShardKeyType.Binary, array); Debug.Assert(result == key); for (int i = 0; i < 10; i++) { Verify(ShardKeyType.Int32); Verify(ShardKeyType.Int64); Verify(ShardKeyType.Guid); Verify(ShardKeyType.Binary); } }