/// <summary> /// Provides O(n) linear weighted rendezvous routing algorithm /// </summary> /// <param name="shards">Shard set out of which one shard is selected</param> /// <param name="shardKey">Sharding key - argument of shard mapping</param> /// <returns>IShard chosen from the shard set</returns> public static IShard RendezvouzRoute(this IEnumerable <IShard> shards, ShardKey shardKey) { //[W]eighted [R]endezvous [H]ashing Algorithm //https://tools.ietf.org/html/draft-mohanty-bess-weighted-hrw-01 var keyHash = shardKey.Hash;//the hash is already "avalanched" shards.IsTrue(s => s != null && s.Any(), "shards!=null && !empty"); IShard best = null; double bestScore = double.MinValue; foreach (var shard in shards) // O(n) { var hash = shard.NameHash ^ keyHash; //both "avalanched" double norm = hash / (double)ulong.MaxValue; //[0.0 .. 1.0] var score = shard.ShardWeight / -Math.Log(norm); //logarithm of real number is negative; log (0) = - infinity ; 1 / -log(0) = 0 if (score > bestScore) { best = shard; bestScore = score; } } return(best); }
public void ShardKey_Ulong() { var v1 = new ShardKey(0ul); var v2 = new ShardKey(0ul); Aver.IsTrue(ShardKey.Type.Ulong == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.AreEqual(v1.ValueUlong, v2.ValueUlong); Aver.IsTrue(v1 == v2); "Value: {0}".SeeArgs(v1);//ToString() v1 = new ShardKey(1234ul); v2 = new ShardKey(1234ul); Aver.IsTrue(ShardKey.Type.Ulong == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.AreEqual(v1.ValueUlong, v2.ValueUlong); Aver.IsTrue(v1 == v2); Aver.AreEqual(1234ul, v1.ValueUlong); "Value: {0}".SeeArgs(v1);//ToString() v2 = new ShardKey(98766554321ul); Aver.AreNotEqual(v1, v2); Aver.AreNotEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreNotEqual(v1.Hash, v2.Hash); Aver.AreNotEqual(v1.ValueUlong, v2.ValueUlong); Aver.IsTrue(v1 != v2); }
/// <summary> /// Validate the object. /// </summary> /// <exception cref="Rest.ValidationException"> /// Thrown if validation fails /// </exception> public virtual void Validate() { if (ShardKey != null) { ShardKey.Validate(); } }
public void ShardKey_Uint() { var v1 = new ShardKey(0u); var v2 = new ShardKey(0u); Aver.IsTrue(ShardKey.Type.Uint == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.AreEqual(v1.ValueUint, v2.ValueUint); Aver.IsTrue(v1 == v2); "Value: {0}".SeeArgs(v1);//ToString() v1 = new ShardKey(1234u); v2 = new ShardKey(1234u); Aver.IsTrue(ShardKey.Type.Uint == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.AreEqual(v1.ValueUint, v2.ValueUint); Aver.IsTrue(v1 == v2); Aver.AreEqual(1234u, v1.ValueUint); "Value: {0}".SeeArgs(v1);//ToString() v2 = new ShardKey(87872u); Aver.AreNotEqual(v1, v2); Aver.AreNotEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreNotEqual(v1.Hash, v2.Hash); Aver.AreNotEqual(v1.ValueUint, v2.ValueUint); Aver.IsTrue(v1 != v2); }
public void TestShardKeyDeserializationAddTrailingZeroes() { foreach (ShardKeyInfo shardKeyInfo in ShardKeyInfo.AllTestShardKeyInfos) { Console.WriteLine(shardKeyInfo); int dataTypeLength = _shardKeyTypeLength[shardKeyInfo.KeyType]; if (shardKeyInfo.RawValue != null && shardKeyInfo.RawValue.Length != dataTypeLength) { // Add trailing zeroes byte[] originalRawValue = shardKeyInfo.RawValue; byte[] rawValueWithTrailingZeroes = new byte[dataTypeLength]; originalRawValue.CopyTo(rawValueWithTrailingZeroes, 0); ShardKey expectedDeserializedShardKey = shardKeyInfo.ShardKeyFromValue; ShardKey actualDeserializedShardKey = ShardKey.FromRawValue(shardKeyInfo.KeyType, rawValueWithTrailingZeroes); // Bug? Below fails when there are trailing zeroes even though the value is Equal //// Verify ShardKey with ShardKey.Equals //Assert.AreEqual( // expectedDeserializedShardKey, // actualDeserializedShardKey); // Bug? Below fails for Binary type if (shardKeyInfo.KeyType != ShardKeyType.Binary) { // Verify ShardKey.Value with value type-specific Equals AssertExtensions.AssertScalarOrSequenceEqual(expectedDeserializedShardKey.Value, actualDeserializedShardKey.Value, null); } } } }
public void ShardKey_Guid() { var v1 = new ShardKey(Guid.Empty); var v2 = new ShardKey(Guid.Empty); Aver.IsTrue(ShardKey.Type.Guid == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.AreEqual(v1.ValueGuid, v2.ValueGuid); Aver.IsTrue(v1 == v2); "Value: {0}".SeeArgs(v1);//ToString() v1 = new ShardKey(Guid.Parse("{F52C9D55-E500-40F4-BF89-39FD6D8D9DC5}")); v2 = new ShardKey(Guid.Parse("{F52C9D55-E500-40F4-BF89-39FD6D8D9DC5}")); Aver.IsTrue(ShardKey.Type.Guid == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.AreEqual(v1.ValueGuid, v2.ValueGuid); Aver.IsTrue(v1 == v2); Aver.AreEqual(Guid.Parse("{F52C9D55-E500-40F4-BF89-39FD6D8D9DC5}"), v1.ValueGuid); "Value: {0}".SeeArgs(v1);//ToString() v2 = new ShardKey(Guid.Parse("{F52C9D55-E500-40F4-BF89-39FD6D8D9DCF}")); Aver.AreNotEqual(v1, v2); Aver.AreNotEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreNotEqual(v1.Hash, v2.Hash); Aver.AreNotEqual(v1.ValueGuid, v2.ValueGuid); Aver.IsTrue(v1 != v2); }
public void String_009(string v) { var h = ShardKey.ForString(v); "Hash: {0:x2}".SeeArgs(h); Aver.AreEqual(new ShardKey(v).Hash, h); }
public void ShardKey_Gdid() { var v1 = new ShardKey(new GDID(0, 0, 0)); var v2 = new ShardKey(new GDID(0, 0, 0)); Aver.IsTrue(ShardKey.Type.Gdid == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.AreEqual(v1.ValueGdid, v2.ValueGdid); Aver.IsTrue(v1 == v2); "Value: {0}".SeeArgs(v1);//ToString() v1 = new ShardKey(new GDID(0, 0, 123)); v2 = new ShardKey(new GDID(0, 0, 123)); Aver.IsTrue(ShardKey.Type.Gdid == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.AreEqual(v1.ValueGdid, v2.ValueGdid); Aver.IsTrue(v1 == v2); Aver.AreEqual(new GDID(0, 0, 123), v1.ValueGdid); "Value: {0}".SeeArgs(v1);//ToString() v2 = new ShardKey(new GDID(0, 0, 124)); Aver.AreNotEqual(v1, v2); Aver.AreNotEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreNotEqual(v1.Hash, v2.Hash); Aver.AreNotEqual(v1.ValueGdid, v2.ValueGdid); Aver.IsTrue(v1 != v2); }
private void TestShardKeyGeneric <TKey>(ShardKeyType keyType, TKey inputValue, Type realType) { // Excercise DetectType // ShardKey k1 = new ShardKey(inputValue); Assert.AreEqual(realType, k1.DataType); // Go to/from raw value ShardKey k2 = new ShardKey(keyType, inputValue); byte[] k2raw = k2.RawValue; ShardKey k3 = ShardKey.FromRawValue(keyType, k2raw); Assert.AreEqual(inputValue, k2.Value); Assert.AreEqual(inputValue, k3.Value); Assert.AreEqual(k2, k3); // verify comparisons Assert.AreEqual(0, k2.CompareTo(k3)); try { k3 = k2.GetNextKey(); Assert.IsTrue(k3 > k2); } catch (InvalidOperationException) { } }
public void ShardKey_EntityId() { var v1 = new ShardKey(new EntityId()); var v2 = new ShardKey(new EntityId()); Aver.IsTrue(ShardKey.Type.EntityId == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.AreEqual(v1.ValueEntityId, v2.ValueEntityId); Aver.IsTrue(v1 == v2); "Value: {0}".SeeArgs(v1);//ToString() v1 = new ShardKey(EntityId.Parse("a@b::c")); v2 = new ShardKey(EntityId.Parse("a@b::c")); Aver.IsTrue(ShardKey.Type.EntityId == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.AreEqual(v1.ValueEntityId, v2.ValueEntityId); Aver.IsTrue(v1 == v2); Aver.AreEqual(EntityId.Parse("a@b::c"), v1.ValueEntityId); "Value: {0}".SeeArgs(v1);//ToString() v2 = new ShardKey(EntityId.Parse("another@b::c")); Aver.AreNotEqual(v1, v2); Aver.AreNotEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreNotEqual(v1.Hash, v2.Hash); Aver.AreNotEqual(v1.ValueEntityId, v2.ValueEntityId); Aver.IsTrue(v1 != v2); }
public void ShardKey_Atom() { var v1 = new ShardKey(new Atom()); var v2 = new ShardKey(new Atom()); Aver.IsTrue(ShardKey.Type.Atom == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.AreEqual(v1.ValueAtom, v2.ValueAtom); Aver.IsTrue(v1 == v2); "Value: {0}".SeeArgs(v1);//ToString() v1 = new ShardKey(Atom.Encode("abc1234")); v2 = new ShardKey(Atom.Encode("abc1234")); Aver.IsTrue(ShardKey.Type.Atom == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.AreEqual(v1.ValueAtom, v2.ValueAtom); Aver.IsTrue(v1 == v2); Aver.AreEqual(Atom.Encode("abc1234"), v1.ValueAtom); "Value: {0}".SeeArgs(v1);//ToString() v2 = new ShardKey(Atom.Encode("4321abc")); Aver.AreNotEqual(v1, v2); Aver.AreNotEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreNotEqual(v1.Hash, v2.Hash); Aver.AreNotEqual(v1.ValueAtom, v2.ValueAtom); Aver.IsTrue(v1 != v2); }
public void TestShardChildSerializationInts() { var sc1 = new ShardKey <int, short>('a', 5, 6, 7); var str = sc1.ToExternalString(); var sc2 = ShardKey <int, short> .FromExternalString(str); sc2.Should().Be(sc1, "because the serialized string creates an equivalent shardChild"); }
public void TestShardKeySerializationMore3() { var sk1 = new ShardKey <DateTime>('a', 0, DateTime.UtcNow); var str = sk1.ToExternalString(); var sk2 = ShardKey <DateTime> .FromExternalString(str); sk2.Should().Be(sk1, "because the serialized DateTime creates an equivalent shardKey"); }
public void TestShardKeySerializationMore1() { var sk1 = new ShardKey <decimal>('a', 0, 4); var str = sk1.ToExternalString(); var sk2 = ShardKey <decimal> .FromExternalString(str); sk2.Should().Be(sk1, "because the serialized decimal creates an equivalent shardKey"); }
public void TestShardKeySerializationMore2() { var sk1 = new ShardKey <Guid>('a', 0, Guid.NewGuid()); var str = sk1.ToExternalString(); var sk2 = ShardKey <Guid> .FromExternalString(str); sk2.Should().Be(sk1, "because the serialized Guid creates an equivalent shardKey"); }
public void TestShardKeySerializationDouble() { var sk1 = new ShardKey <double>('a', 0, 0.3); var str = sk1.ToExternalString(); var sk2 = ShardKey <double> .FromExternalString(str); sk2.Should().Be(sk1, "because the serialized double creates an equivalent shardKey"); }
public void TestShardKeySerializationInts2() { var sk1 = new ShardKey <long>('a', 3, 4); var str = sk1.ToExternalString(); var sk2 = ShardKey <long> .FromExternalString(str); sk2.Should().Be(sk1, "because the serialized string creates an equivalent shardKey"); }
protected internal Shard(ShardSet set, IConfigSectionNode conf) { m_Set = set.NonNull(nameof(set)); ConfigAttribute.Apply(this, conf.NonEmpty(nameof(conf))); m_Name = conf.ValOf(Configuration.CONFIG_NAME_ATTR).NonBlank("$name"); m_NameHash = ShardKey.ForString(m_Name); m_Weight = conf.Of(CONFIG_WEIGHT_ATTR).ValueAsDouble(1.0d); m_ConnectString = conf.Of(CONFIG_CONNECT_ATTR, CONFIG_CONNECT_STRING_ATTR).Value.NonBlank($"${CONFIG_CONNECT_ATTR}"); m_DatabaseName = conf.Of(CONFIG_DB_ATTR).Value.NonBlank($"${CONFIG_DB_ATTR}"); }
/// <summary> /// Validate the object. /// </summary> /// <exception cref="ValidationException"> /// Thrown if validation fails /// </exception> public override void Validate() { base.Validate(); if (DatabaseName == null) { throw new ValidationException(ValidationRules.CannotBeNull, "DatabaseName"); } if (ShardKey != null) { ShardKey.Validate(); } }
public void GDID_000(GDID v) { var h = new ShardKey(v); "GDID Hash: {0:x2}".SeeArgs(h.Hash); Aver.AreEqual(new ShardKey(v).Hash, h.Hash); Aver.AreEqual(new ShardKey(v), h); Aver.IsTrue(new ShardKey(v).Equals(h)); Aver.IsTrue(new ShardKey(v) == h); Aver.IsFalse(new ShardKey(v) != h); Aver.IsTrue(new ShardKey(v).GetHashCode() == h.GetHashCode()); }
/// <summary> /// Maps sharding ID to processor ID. This method is used to map process unqiue mutex id, or GDID to processor ID - which denotes a /// set of actual executor hosts /// </summary> public int MapShardingKeyToProcessorID(string shardingKey) { var ids = ProcessorMap.Keys.ToArray(); if (ids.Length == 0) { throw new MetabaseException("TODO: MapShardingKeyToProcessorID(ProcessorMap is empty)"); } var hash = new ShardKey(shardingKey).GetDistributedStableHash(); var idx = hash % (ulong)ids.Length; return(ids[idx]); }
/// <summary> /// Helper function to advance mapping iterators. /// </summary> /// <param name="iterator">The iterator to advance.</param> /// <param name="keyType">The data type of the map key.</param> /// <param name="nextMapping">Output value that will contain next mapping.</param> /// <param name="nextRange">Output value that will contain next range.</param> /// <param name="nextMinKey">Output value that will contain next min key.</param> private static void MoveToNextMapping( IEnumerator <IStoreMapping> iterator, ShardKeyType keyType, out IStoreMapping nextMapping, out ShardRange nextRange, out ShardKey nextMinKey) { nextMapping = iterator.MoveNext() ? iterator.Current : null; nextRange = nextMapping != null ? new ShardRange( ShardKey.FromRawValue(keyType, nextMapping.MinValue), ShardKey.FromRawValue(keyType, nextMapping.MaxValue)) : null; nextMinKey = nextRange != null ? nextRange.Low : null; }
/// <summary> /// Helper function that produces a list of MappingComparisonResults from union of points in the gsmMappings and lsmMappings. /// </summary> /// <param name="ssm">StoreShardmap to be referenced in produced MappingComparisonResults</param> /// <param name="gsmMappings">List of mappings from the GSM.</param> /// <param name="lsmMappings">List of mappings from the LSM.</param> /// <returns>List of mappingcomparisonresults: one for each range arising from the union of boundaries in gsmMappings and lsmMappings.</returns> internal static List <MappingComparisonResult> ComparePointMappings( IStoreShardMap ssm, IEnumerable <IStoreMapping> gsmMappings, IEnumerable <IStoreMapping> lsmMappings) { ShardKeyType keyType = ssm.KeyType; // Get a Linq-able set of points from the input mappings. // IDictionary <ShardKey, IStoreMapping> gsmPoints = gsmMappings.ToDictionary(gsmMapping => ShardKey.FromRawValue(keyType, gsmMapping.MinValue)); IDictionary <ShardKey, IStoreMapping> lsmPoints = lsmMappings.ToDictionary(lsmMapping => ShardKey.FromRawValue(keyType, lsmMapping.MinValue)); // Construct the output list. This is the concatenation of 3 mappings: // 1.) Intersection (the key exists in both the shardmap and the shard.) // 2.) Shard only (the key exists only in the shard.) // 3.) Shardmap only (the key exists only in the shardmap.) // List <MappingComparisonResult> results = (new List <MappingComparisonResult>()).Concat( // Intersection. lsmPoints.Keys.Intersect(gsmPoints.Keys).Select( commonPoint => new MappingComparisonResult( ssm, new ShardRange(commonPoint, commonPoint.GetNextKey()), MappingLocation.MappingInShardMapAndShard, gsmPoints[commonPoint], lsmPoints[commonPoint])) ).Concat( // Lsm only. lsmPoints.Keys.Except(gsmPoints.Keys).Select( lsmOnlyPoint => new MappingComparisonResult( ssm, new ShardRange(lsmOnlyPoint, lsmOnlyPoint.GetNextKey()), MappingLocation.MappingInShardOnly, null, lsmPoints[lsmOnlyPoint])) ).Concat( // Gsm only. gsmPoints.Keys.Except(lsmPoints.Keys).Select( gsmOnlyPoint => new MappingComparisonResult( ssm, new ShardRange(gsmOnlyPoint, gsmOnlyPoint.GetNextKey()), MappingLocation.MappingInShardMapOnly, gsmPoints[gsmOnlyPoint], null))).ToList(); return(results); }
/// <summary> /// Orchestrates an Mongo/s call to a remote service pointed to by a logical `remoteAddress`. /// This algorithm provides sharding, fail-over and circuit breaker functionality built-in. /// An actual call is performed in a passed-in call body functor. /// </summary> /// <typeparam name="TResult">The resulting type of the call, as obtained from call body</typeparam> /// <param name="service">A service to run against</param> /// <param name="remoteAddress">Logical address of the remote service, not to be confused with physical. Physical addressed are resolved from logical addresses</param> /// <param name="contract">Logical contract name, such as * for any contract</param> /// <param name="shardKey">A key value used for sharding (such as GDID, GUID) traffic</param> /// <param name="body">Call body functor. May not be null</param> /// <param name="cancellation">Optional CancellationToken</param> /// <param name="network">Logical network name used for endpoint address resolution, e.g. this is used to segregate traffic by physical channels</param> /// <param name="binding">Logical binding (sub-protocol) name (e.g. json/bix)</param> /// <returns>TResult call result or throws `ClientException` if call eventually failed after all failovers tried</returns> public static TResult CallSync <TResult>(this IMongoDbService service, string remoteAddress, string contract, ShardKey shardKey, Func <IMongoDbTransport, CancellationToken?, TResult> body, CancellationToken?cancellation = null, Atom?network = null, Atom?binding = null) { body.NonNull(nameof(body)); var assignments = service.NonNull(nameof(service)) .GetEndpointsForCall(remoteAddress, contract, shardKey, network, binding); return(assignments.CallSync(body, cancellation)); }
/// <summary> /// Orchestrates an Http/s call to a remote service pointed to by a logical `remoteAddress`. /// This algorithm provides sharding, fail-over and circuit breaker functionality built-in. /// An actual call is performed in a passed-in call body functor. /// </summary> /// <typeparam name="TResult">The resulting type of the call, as obtained from call body</typeparam> /// <param name="service">A service to run against</param> /// <param name="remoteAddress">Logical address of the remote service, not to be confused with physical. Physical addressed are resolved from logical addresses</param> /// <param name="contract">Logical contract name, such as * for any contract</param> /// <param name="shardKey">A key value used for sharding (such as GDID, GUID) traffic</param> /// <param name="body">Call body functor. May not be null</param> /// <param name="cancellation">Optional CancellationToken</param> /// <param name="network">Logical network name used for endpoint address resolution, e.g. this is used to segregate traffic by physical channels</param> /// <param name="binding">Logical binding (sub-protocol) name (e.g. json/bix)</param> /// <returns>TResult call result or throws `ClientException` if call eventually failed after all fail-overs tried</returns> public static async Task <TResult> Call <TResult>(this IHttpService service, string remoteAddress, string contract, ShardKey shardKey, Func <IHttpTransport, CancellationToken?, Task <TResult> > body, CancellationToken?cancellation = null, Atom?network = null, Atom?binding = null) { body.NonNull(nameof(body)); var assignments = service.NonNull(nameof(service)) .GetEndpointsForCall(remoteAddress, contract, shardKey, network, binding); return(await assignments.Call(body, cancellation).ConfigureAwait(false)); }
public void ShardKey_ByteArray() { var v1 = new ShardKey((byte[])null); var v2 = new ShardKey((byte[])null); Aver.IsTrue(ShardKey.Type.ByteArray == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.IsTrue(v1.ValueByteArray.MemBufferEquals(v2.ValueByteArray)); Aver.IsTrue(v1 == v2); "Value: {0}".SeeArgs(v1);//ToString() v1 = new ShardKey(new byte[0]); v2 = new ShardKey(new byte[0]); Aver.IsTrue(ShardKey.Type.ByteArray == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.IsTrue(v1.ValueByteArray.MemBufferEquals(v2.ValueByteArray)); Aver.IsTrue(v1 == v2); "Value: {0}".SeeArgs(v1);//ToString() v1 = new ShardKey(new byte[] { 1, 3, 9 }); v2 = new ShardKey(new byte[] { 1, 3, 9 }); Aver.IsTrue(ShardKey.Type.ByteArray == v1.DataType); Aver.AreEqual(v1, v2); Aver.AreEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreEqual(v1.Hash, v2.Hash); Aver.IsTrue(v1.ValueByteArray.MemBufferEquals(v2.ValueByteArray)); Aver.IsTrue(v1 == v2); Aver.AreEqual(3, v1.ValueByteArray.Length); Aver.AreEqual(1, v1.ValueByteArray[0]); Aver.AreEqual(3, v1.ValueByteArray[1]); Aver.AreEqual(9, v1.ValueByteArray[2]); "Value: {0}".SeeArgs(v1);//ToString() v2 = new ShardKey(new byte[] { 1, 3, 9, 11 }); Aver.AreNotEqual(v1, v2); Aver.AreNotEqual(v1.GetHashCode(), v2.GetHashCode()); Aver.AreNotEqual(v1.Hash, v2.Hash); Aver.IsFalse(v1.ValueByteArray.MemBufferEquals(v2.ValueByteArray)); Aver.IsTrue(v1 != v2); }
private void Verify(ShardKeyType kind) { byte[] bytes = null; ShardKey key = null; ShardKey result = null; switch (kind) { case ShardKeyType.Int32: bytes = new byte[sizeof(int)]; rValGen.NextBytes(bytes); Int32 int32 = BitConverter.ToInt32(bytes, 0); key = new ShardKey(int32); result = key.GetNextKey(); Debug.Assert(result.IsMax || result == new ShardKey(int32 + 1)); break; case ShardKeyType.Int64: bytes = new byte[sizeof(long)]; rValGen.NextBytes(bytes); Int64 int64 = BitConverter.ToInt64(bytes, 0); key = new ShardKey(int64); result = key.GetNextKey(); Debug.Assert(result.IsMax || result == new ShardKey(int64 + 1)); break; case ShardKeyType.Guid: Guid guid = Guid.NewGuid(); key = new ShardKey(guid); result = key.GetNextKey(); // verify only the API call break; case ShardKeyType.Binary: bytes = new byte[128]; rValGen.NextBytes(bytes); key = new ShardKey(bytes); result = key.GetNextKey(); // verify only the API call break; default: throw new ArgumentOutOfRangeException( "kind", kind, Errors._ShardKey_UnsupportedShardKeyType); } }
public void TestShardKeyWithDateTimeOffset() { DateTimeOffset testValue = DateTimeOffset.Now; TestShardKeyGeneric <DateTimeOffset>(ShardKeyType.DateTimeOffset, testValue, typeof(DateTimeOffset)); DateTime d1 = DateTime.Now; ShardKey k1 = new ShardKey(new DateTimeOffset(d1, DateTimeOffset.Now.Offset)); ShardKey k2 = new ShardKey(new DateTimeOffset(d1.ToUniversalTime(), TimeSpan.FromHours(0))); Assert.AreEqual(k1, k2); ShardKey k3 = ShardKey.MinDateTimeOffset; Assert.AreNotEqual(k1, k3); }
public void ShardKey_String_Collision() { var set = new HashSet <ulong>(); var dup = 0; for (var i = 0; i < 1_000_000; i++) { var k = new ShardKey("some-{0}".Args(i)); if (!set.Add(k.Hash)) { dup++; } } dup.See(); Aver.AreEqual(0, dup); }
public void ShardKey_Gdid_Collision() { var set = new HashSet <ulong>(); var dup = 0; for (var i = 0; i < 1_000_000; i++) { var k = new ShardKey(new GDID(0, (ulong)i)); if (!set.Add(k.Hash)) { dup++; } } dup.See(); Aver.AreEqual(0, dup); }
/// <summary> /// Constructs request for obtaining mapping from GSM based on given key. /// </summary> /// <param name="shardMapManager">Shard map manager.</param> /// <param name="operationName">Operation being executed.</param> /// <param name="shardMap">Local shard map.</param> /// <param name="key">Key for lookup operation.</param> /// <param name="policy">Policy for cache update.</param> /// <param name="errorCategory">Error category.</param> /// <param name="cacheResults">Whether to cache the results of the operation.</param> /// <param name="ignoreFailure">Ignore shard map not found error.</param> protected internal FindMappingByKeyGlobalOperation( ShardMapManager shardMapManager, string operationName, IStoreShardMap shardMap, ShardKey key, CacheStoreMappingUpdatePolicy policy, ShardManagementErrorCategory errorCategory, bool cacheResults, bool ignoreFailure) : base(shardMapManager.Credentials, shardMapManager.RetryPolicy, operationName) { _manager = shardMapManager; _shardMap = shardMap; _key = key; _policy = policy; _errorCategory = errorCategory; _cacheResults = cacheResults; _ignoreFailure = ignoreFailure; }
/// <summary> /// Finds range mapping which contain the given key for the given ShardMap. /// </summary> /// <param name="ssm">Shard map to find mappings in.</param> /// <param name="shardKey">ShardKey being searched.</param> /// <returns>Storage operation result.</returns> public virtual IStoreResults FindRangeMappingByKeyGlobal(IStoreShardMap ssm, ShardKey shardKey) { try { return SqlStore.FindMappingByKeyGlobalHelper(ssm, shardKey.RawValue, @"__ShardManagement.smm_findRangeMappingByKeyGlobal"); } catch (SqlException se) { throw new StoreException( Errors.SqlStore_ShardMap_AddRemoveUpdateFindMappingGlobal_SqlException, se, "FindMappingByKey", "Range", ssm.Name); } }
/// <summary> /// Request to get mapping from LSM for a particular key belonging to a shard map. /// </summary> /// <param name="shardMap">Shard map whose mappings are being requested.</param> /// <param name="key">Key being searched.</param> /// <returns>Xml formatted request.</returns> internal static XElement FindShardMappingByKeyLocal( IStoreShardMap shardMap, ShardKey key) { return new XElement( @"FindShardMappingByKeyLocal", StoreOperationRequestBuilder.s_lsmVersion, StoreObjectFormatterXml.WriteIStoreShardMap("ShardMap", shardMap), StoreObjectFormatterXml.WriteShardKey(key)); }
/// <summary> /// Looks up a mapping by key. /// </summary> /// <param name="key">Key value.</param> /// <param name="sm">Storage mapping object.</param> /// <returns>Mapping object which has the key value.</returns> internal override ICacheStoreMapping LookupByKey(ShardKey key, out IStoreMapping sm) { CacheMapping cm; _mappingsByKey.TryGetValue(key, out cm); if (cm != null) { sm = cm.Mapping; } else { sm = null; } return cm; }
/// <summary> /// Converts a given shard key to xml. /// </summary> /// <param name="key">Input key.</param> /// <returns>XElement representing the given key.</returns> internal static XElement WriteShardKey(ShardKey key) { Debug.Assert(key != null); return new XElement("Key", new XElement("Value", StringUtils.ByteArrayToString(key.RawValue))); }
/// <summary> /// Performs binary search on the cached mappings and returns the /// index of mapping object whose min-value is less than and closest /// to given key value. /// </summary> /// <param name="key">Input key.</param> /// <returns>Index of range in the cache which contains the given key.</returns> private int GetIndexOfMappingWithClosestMaxGreaterThanOrEqualToMaxKey(ShardKey key) { IList<ShardRange> rangeKeys = _mappingsByRange.Keys; int lb = 0; int ub = rangeKeys.Count - 1; while (lb <= ub) { int mid = lb + (ub - lb) / 2; ShardRange current = rangeKeys[mid]; if (current.High > key) { if (current.Low <= key) { return mid; } else { ub = mid - 1; } } else { lb = mid + 1; } } return lb; }
/// <summary> /// Looks up a given key in given shard map. /// </summary> /// <param name="shardMap">Storage representation of shard map.</param> /// <param name="key">Key value.</param> /// <returns>Mapping corresponding to <paramref name="key"/> or null.</returns> public virtual ICacheStoreMapping LookupMappingByKey(IStoreShardMap shardMap, ShardKey key) { ICacheStoreMapping sm = null; using (ReadLockScope rls = _cacheRoot.GetReadLockScope(false)) { CacheShardMap csm = _cacheRoot.LookupById(shardMap.Id); if (csm != null) { using (ReadLockScope rlsShardMap = csm.GetReadLockScope(false)) { IStoreMapping smDummy; sm = csm.Mapper.LookupByKey(key, out smDummy); } } } return sm; }
/// <summary> /// Finds point mapping which contain the given range for the shard map. /// </summary> /// <param name="ssm">Shard map to find mappings in.</param> /// <param name="key">Key to which a corresponding mapping should be found.</param> /// <param name="shard">Shard container requiring Id and ShardLocation.</param> /// <returns>Storage operation result.</returns> public virtual IStoreResults FindPointMappingByKeyLocal(IStoreShardMap ssm, ShardKey key, IStoreShard shard) { try { return SqlStore.FindMappingByKeyLocalHelper(ssm, key, shard); } catch (SqlException se) { throw new StoreException( Errors.SqlStore_ShardMap_AddRemoveUpdateFindMappingLocal_SqlException, se, "FindMappingForKey", "Point", shard.Location); } }
/// <summary> /// Looks up a mapping by key. /// </summary> /// <param name="key">Key value.</param> /// <param name="sm">Storage mapping object.</param> /// <returns>Mapping object which has the key value.</returns> internal abstract ICacheStoreMapping LookupByKey(ShardKey key, out IStoreMapping sm);
public void ShardKeyTests() { ShardKey key = null; ShardKey result = null; byte[] array = null; byte[] arraymax = null; // Verify boundary conditions result = maxNonNullKey32.GetNextKey(); Debug.Assert(result.IsMax); Debug.Assert(result == ShardKey.MaxInt32); result = maxNonNullKey64.GetNextKey(); Debug.Assert(result.IsMax); Debug.Assert(result == ShardKey.MaxInt64); array = Enumerable.Repeat((byte)0xff, 16).ToArray(); key = ShardKey.FromRawValue(ShardKeyType.Guid, array); // can not use other ctor because normalized representation differ result = key.GetNextKey(); Debug.Assert(result.IsMax); Debug.Assert(result == ShardKey.MaxGuid); array = Enumerable.Repeat((byte)0xff, 128).ToArray(); key = new ShardKey(array); result = key.GetNextKey(); Debug.Assert(result.IsMax); Debug.Assert(result == ShardKey.MaxBinary); key = new ShardKey(max32 - 1); result = key.GetNextKey(); Debug.Assert(result == maxNonNullKey32); key = new ShardKey(max64 - 1); result = key.GetNextKey(); Debug.Assert(result == maxNonNullKey64); arraymax = Enumerable.Repeat((byte)0xff, 16).ToArray(); array = Enumerable.Repeat((byte)0xff, 16).ToArray(); array[15] = 0xfe; key = ShardKey.FromRawValue(ShardKeyType.Guid, array); // can not use other ctor because normalized representation differ result = key.GetNextKey(); Debug.Assert(result == ShardKey.FromRawValue(ShardKeyType.Guid, arraymax)); arraymax = Enumerable.Repeat((byte)0xff, 128).ToArray(); array = Enumerable.Repeat((byte)0xff, 128).ToArray(); array[127] = 0xfe; key = new ShardKey(array); result = key.GetNextKey(); Debug.Assert(result == ShardKey.FromRawValue(ShardKeyType.Binary, arraymax)); key = new ShardKey(ShardKeyType.Int32, null); AssertExtensions.AssertThrows<InvalidOperationException>(() => key.GetNextKey()); key = new ShardKey(ShardKeyType.Int64, null); AssertExtensions.AssertThrows<InvalidOperationException>(() => key.GetNextKey()); key = new ShardKey(ShardKeyType.Guid, null); AssertExtensions.AssertThrows<InvalidOperationException>(() => key.GetNextKey()); key = new ShardKey(ShardKeyType.Binary, null); AssertExtensions.AssertThrows<InvalidOperationException>(() => key.GetNextKey()); result = ShardKey.MinInt32.GetNextKey(); Debug.Assert(result == new ShardKey(Int32.MinValue + 1)); result = ShardKey.MinInt64.GetNextKey(); Debug.Assert(result == new ShardKey(Int64.MinValue + 1)); result = ShardKey.MinGuid.GetNextKey(); array = new byte[16]; array[15] = 0x01; key = ShardKey.FromRawValue(ShardKeyType.Guid, array); Debug.Assert(result == key); result = ShardKey.MinBinary.GetNextKey(); array = new byte[128]; array[127] = 0x01; key = ShardKey.FromRawValue(ShardKeyType.Binary, array); Debug.Assert(result == key); for (int i = 0; i < 10; i++) { Verify(ShardKeyType.Int32); Verify(ShardKeyType.Int64); Verify(ShardKeyType.Guid); Verify(ShardKeyType.Binary); } }
/// <summary> /// Helper function to advance mapping iterators. /// </summary> /// <param name="iterator">The iterator to advance.</param> /// <param name="keyType">The data type of the map key.</param> /// <param name="nextMapping">Output value that will contain next mapping.</param> /// <param name="nextRange">Output value that will contain next range.</param> /// <param name="nextMinKey">Output value that will contain next min key.</param> private static void MoveToNextMapping( IEnumerator<IStoreMapping> iterator, ShardKeyType keyType, out IStoreMapping nextMapping, out ShardRange nextRange, out ShardKey nextMinKey) { nextMapping = iterator.MoveNext() ? iterator.Current : null; nextRange = nextMapping != null ? new ShardRange( ShardKey.FromRawValue(keyType, nextMapping.MinValue), ShardKey.FromRawValue(keyType, nextMapping.MaxValue)) : null; nextMinKey = nextRange != null ? nextRange.Low : null; }
/// <summary> /// Looks up a mapping by key. /// </summary> /// <param name="key">Key value.</param> /// <param name="sm">Storage mapping object.</param> /// <returns>Mapping object which has the key value.</returns> internal override ICacheStoreMapping LookupByKey(ShardKey key, out IStoreMapping sm) { CacheMapping cm; // Performs a binary search in the ranges for key value and // then return the result. int rangeIndex = this.GetIndexOfMappingContainingShardKey(key); if (rangeIndex != -1) { ShardRange range = _mappingsByRange.Keys[rangeIndex]; cm = _mappingsByRange[range]; // DEVNOTE(wbasheer): We should clone the mapping. sm = cm.Mapping; } else { cm = null; sm = null; } return cm; }
/// <summary> /// Performs binary search on the cached mappings and returns the /// index of mapping object which contains the given key. /// </summary> /// <param name="key">Input key.</param> /// <returns>Index of range in the cache which contains the given key.</returns> private int GetIndexOfMappingContainingShardKey(ShardKey key) { IList<ShardRange> rangeKeys = _mappingsByRange.Keys; int lb = 0; int ub = rangeKeys.Count - 1; while (lb <= ub) { int mid = lb + (ub - lb) / 2; ShardRange current = rangeKeys[mid]; if (current.Contains(key)) { return mid; } else if (key < current.Low) { ub = mid - 1; } else { lb = mid + 1; } } return -1; }
/// <summary> /// Splits given range mapping into 2 new mappings. /// </summary> /// <param name="ssm">Shard map to find mappings in.</param> /// <param name="sm">Store mapping to split.</param> /// <param name="splitPoint">Split point in the mapping.</param> /// <param name="lockOwnerId">Lock owner id of this mapping</param> /// <returns>Storage operation result.</returns> public virtual IStoreResults SplitRangeMappingGlobal(IStoreShardMap ssm, IStoreMapping sm, ShardKey splitPoint, Guid lockOwnerId) { try { SqlResults result = new SqlResults(); using (SqlCommand cmd = SqlTransactionScopeGlobal.CreateSqlCommand()) { cmd.CommandText = @"__ShardManagement.smm_splitRangeShardMappingGlobal"; cmd.CommandType = CommandType.StoredProcedure; XElement input = new XElement(cmd.CommandText, new XElement("gsm_version", SqlStoreGsmVersion.ToString()), new XElement("sm_id", ssm.Id.ToString()), new XElement("lo_id", lockOwnerId.ToString()), new XElement("sm_version", ssm.Version.ToString()), new XElement("m_id", sm.Id.ToString()), new XElement("m_version", sm.Version.ToString()), new XElement("split_point", StringUtils.ByteArrayToString(splitPoint.RawValue)) ); SqlStore.AddCommandParameter(cmd, "@input", SqlDbType.Xml, ParameterDirection.Input, 0, input.ToString()); SqlParameter resultParam = SqlStore.AddCommandParameter(cmd, "@result", SqlDbType.Int, ParameterDirection.Output, 0, 0); using (SqlDataReader reader = cmd.ExecuteReader()) { result.Fetch(reader); } // Output parameter will be used to specify the outcome. result.Result = (StoreResult)resultParam.Value; } return result; } catch (SqlException se) { throw new StoreException( Errors.SqlStore_RangeShardMap_SplitMerge_SqlException, se, "Split", ssm.Name); } }
/// <summary> /// Looks up a given key in given shard map. /// </summary> /// <param name="shardMap">Storage representation of shard map.</param> /// <param name="key">Key value.</param> /// <returns>Mapping corresponding to <paramref name="key"/> or null.</returns> public virtual ICacheStoreMapping LookupMappingByKey(IStoreShardMap shardMap, ShardKey key) { ICacheStoreMapping sm = null; using (ReadLockScope rls = _cacheRoot.GetReadLockScope(false)) { CacheShardMap csm = _cacheRoot.LookupById(shardMap.Id); if (csm != null) { using (ReadLockScope rlsShardMap = csm.GetReadLockScope(false)) { IStoreMapping smDummy; sm = csm.Mapper.LookupByKey(key, out smDummy); // perf counter can not be updated in csm.Mapper.LookupByKey() as this function is also called from csm.Mapper.AddOrUpdate() // so updating perf counter value here instead. csm.IncrementPerformanceCounter(sm == null ? PerformanceCounterName.MappingsLookupFailedPerSec : PerformanceCounterName.MappingsLookupSucceededPerSec); } } } return sm; }
/// <summary> /// Constructs request for obtaining mapping from GSM based on given key. /// </summary> /// <param name="shardMapManager">Shard map manager.</param> /// <param name="operationName">Operation being executed.</param> /// <param name="shardMap">Local shard map.</param> /// <param name="key">Key for lookup operation.</param> /// <param name="policy">Policy for cache update.</param> /// <param name="errorCategory">Error category.</param> /// <param name="cacheResults">Whether to cache the results of the operation.</param> /// <param name="ignoreFailure">Ignore shard map not found error.</param> /// <returns>The store operation.</returns> public virtual IStoreOperationGlobal CreateFindMappingByKeyGlobalOperation( ShardMapManager shardMapManager, string operationName, IStoreShardMap shardMap, ShardKey key, CacheStoreMappingUpdatePolicy policy, ShardManagementErrorCategory errorCategory, bool cacheResults, bool ignoreFailure) { return new FindMappingByKeyGlobalOperation( shardMapManager, operationName, shardMap, key, policy, errorCategory, cacheResults, ignoreFailure); }
/// <summary> /// find mappings in Local ShardMap. /// </summary> /// <param name="ssm">Shard map to find mappings in.</param> /// <param name="key">Key corresponding to a point mapping.</param> /// <param name="shard">Shard to find mappings in.</param> /// <returns>Storage operation result.</returns> private static IStoreResults FindMappingByKeyLocalHelper(IStoreShardMap ssm, ShardKey key, IStoreShard shard) { string storedProcName = "__ShardManagement.smm_getPointShardMappingLocal"; SqlResults result = new SqlResults(); using (SqlCommand cmd = SqlTransactionScopeLocal.CreateSqlCommand(shard.Location)) { cmd.CommandText = storedProcName; cmd.CommandType = CommandType.StoredProcedure; XElement input = new XElement(storedProcName, new XElement("lsm_version", SqlStoreLsmVersion.ToString()), new XElement("sm_id", ssm.Id.ToString()), new XElement("s_id", shard.Id.ToString()), new XElement("m_value", StringUtils.ByteArrayToString(key.RawValue)) ); SqlStore.AddCommandParameter(cmd, "@input", SqlDbType.Xml, ParameterDirection.Input, 0, input.ToString()); SqlParameter resultParam = SqlStore.AddCommandParameter(cmd, "@result", SqlDbType.Int, ParameterDirection.Output, 0, 0); using (SqlDataReader reader = cmd.ExecuteReader()) { result.Fetch(reader); } // Output parameter will be used to specify the outcome. result.Result = (StoreResult)resultParam.Value; } return result; }
public virtual ICacheStoreMapping LookupMappingByKey(IStoreShardMap shardMap, ShardKey key) { return this.inner.LookupMappingByKey(shardMap, key); }
/// <summary> /// Sets the stub of CacheStore.LookupMappingByKey(IStoreShardMap shardMap, ShardKey key) /// </summary> public override ICacheStoreMapping LookupMappingByKey(IStoreShardMap shardMap, ShardKey key) { Func<IStoreShardMap, ShardKey, ICacheStoreMapping> func1 = this.LookupMappingByKeyIStoreShardMapShardKey; if (func1 != null) return func1(shardMap, key); if (this.___callBase) return base.LookupMappingByKey(shardMap, key); return this.InstanceBehavior.Result<StubCacheStore, ICacheStoreMapping>(this, "LookupMappingByKey"); }
/// <summary> /// Initializes a new instance /// </summary> public StubFindMappingByKeyGlobalOperation(ShardMapManager shardMapManager, string operationName, IStoreShardMap shardMap, ShardKey key, CacheStoreMappingUpdatePolicy policy, ShardManagementErrorCategory errorCategory, bool cacheResults, bool ignoreFailure) : base(shardMapManager, operationName, shardMap, key, policy, errorCategory, cacheResults, ignoreFailure) { this.InitializeStub(); }
/// <summary> /// Sets the stub of StoreOperationFactory.CreateFindMappingByKeyGlobalOperation(ShardMapManager shardMapManager, String operationName, IStoreShardMap shardMap, ShardKey key, CacheStoreMappingUpdatePolicy policy, ShardManagementErrorCategory errorCategory, Boolean cacheResults, Boolean ignoreFailure) /// </summary> public override IStoreOperationGlobal CreateFindMappingByKeyGlobalOperation(ShardMapManager shardMapManager, string operationName, IStoreShardMap shardMap, ShardKey key, CacheStoreMappingUpdatePolicy policy, ShardManagementErrorCategory errorCategory, bool cacheResults, bool ignoreFailure) { Func<ShardMapManager, string, IStoreShardMap, ShardKey, CacheStoreMappingUpdatePolicy, ShardManagementErrorCategory, bool, bool, IStoreOperationGlobal> func1 = this.CreateFindMappingByKeyGlobalOperationShardMapManagerStringIStoreShardMapShardKeyCacheStoreMappingUpdatePolicyShardManagementErrorCategoryBooleanBoolean; if (func1 != null) return func1(shardMapManager, operationName, shardMap, key, policy, errorCategory, cacheResults, ignoreFailure); if (this.___callBase) return base.CreateFindMappingByKeyGlobalOperation(shardMapManager, operationName, shardMap, key, policy, errorCategory, cacheResults, ignoreFailure); return this.InstanceBehavior.Result<StubStoreOperationFactory, IStoreOperationGlobal>(this, "CreateFindMappingByKeyGlobalOperation"); }
public override ICacheStoreMapping LookupMappingByKey(IStoreShardMap shardMap, ShardKey key) { this.LookupMappingCount++; ICacheStoreMapping result = base.LookupMappingByKey(shardMap, key); if (result == null) { this.LookupMappingMissCount++; } else { this.LookupMappingHitCount++; } return result; }