/// <summary> /// Map a element to its corresponding bits in the array. /// </summary> /// <param name="element">The element to be mapped to indexes</param> /// <param name="numberOfIndexesRequested">The number of indexes to associate with the element. If not set, /// uses the MaximumBitIndexesPerElement specified in the constructor.</param> /// <returns>The array indexes associated with the element.</returns> public IEnumerable <int> GetIndexesAssociatedWithAnElement(byte[] element, int?numberOfIndexesRequested = null) { // Build a set of indexes into the bit array associated with the element HashSet <int> indexesIntoBitArray = new HashSet <int>(); // Determine how many indexes to generate for this element. // The lesser of the maximum number supported and the number requested (if provided) int numberOfBitIndexesToCreate = Math.Min(HashFunctionsMappingElementsToBitsInTheArray.Length, numberOfIndexesRequested ?? int.MaxValue); // Use one hash function to generate each index for (int i = 0; i < numberOfBitIndexesToCreate; i++) { UniversalHashFunction hashFunction = HashFunctionsMappingElementsToBitsInTheArray[i]; byte[] valueToHash = ManagedSHA256.Hash(element); do { // Use the hash function to index into the array of bits int indexIntoBitArray = (int)(hashFunction.Hash(valueToHash) % (uint)Length); if (indexesIntoBitArray.Add(indexIntoBitArray)) { // Common case: this index points to a bit that is not yet associated with the element // This index can thus represent the i_th bit for this element. break; } // This hash function generated an index to a bit that is already associated with this element. // We'll need to rehash to create an index for the i_th bit that was not already assigned. valueToHash = ManagedSHA256.Hash(valueToHash); } while (true); } return(indexesIntoBitArray); }
/// <summary> /// Construct a filter array. /// </summary> /// <param name="numberOfBitsInArray">The size of the array in bits.</param> /// <param name="maximumBitIndexesPerElement">The maximum (and default) number of indexes (bits) in the array to associate with elements.</param> /// <param name="initilizeBitsOfArrayAtRandom">If set to true, the bits of the filter array will be set to 0 or 1 at random (indpendently, each with probability 0.5).</param> /// <param name="saltForHashFunctions">A salt used to generate the hash functions. /// Any two filter arrays generated with the same salt will use the same hash functions. /// The salt should be kept secret from attackerse who might try to manipulate the selection of elements, /// such as to intentionally cause bit collisions with the array.</param> public FilterArray(int numberOfBitsInArray, int maximumBitIndexesPerElement, bool initilizeBitsOfArrayAtRandom, string saltForHashFunctions = "") { // Align on byte boundary to guarantee no less than numberOfBitsInArray int capacityInBytes = (numberOfBitsInArray + 7) / 8; // Create hash functions to map elements to indexes in the bit array. HashFunctionsMappingElementsToBitsInTheArray = new UniversalHashFunction[maximumBitIndexesPerElement]; for (int i = 0; i < HashFunctionsMappingElementsToBitsInTheArray.Length; i++) { HashFunctionsMappingElementsToBitsInTheArray[i] = new UniversalHashFunction(i + ":" + saltForHashFunctions, 64); } if (initilizeBitsOfArrayAtRandom) { // Initialize the bit array setting ~half the bits randomly to zero by using the // cryptographic random number generator. byte[] initialBitValues = new byte[capacityInBytes]; StrongRandomNumberGenerator.GetBytes(initialBitValues); BitArray = new BitArray(initialBitValues); } else { // Start with all bits of the array set to zero. BitArray = new BitArray(capacityInBytes * 8); } }
/// <summary> /// Create a hash ring /// </summary> /// <param name="key">A key used for creating hash functions. The key must be kept secret /// from adversaries that might attempt to perform algorithmic complexity attacks to do such /// things as ensuring all the important values that one might hash map to the same member.</param> /// <param name="initialMembers">An optional set of members to place onto the ring.</param> /// <param name="maxInputLengthInBytesForUniversalHashFunction"></param> /// <param name="numberOfPointsOnRingForEachMember"></param> public ConsistentHashRing(string key, IEnumerable <KeyValuePair <string, TMember> > initialMembers = null, int maxInputLengthInBytesForUniversalHashFunction = DefaultMaxInputLengthInBytesForUniversalHashFunction, int numberOfPointsOnRingForEachMember = DefaultNumberOfPointsOnRingForEachMember) { _numberOfPointsOnRingForEachMember = numberOfPointsOnRingForEachMember; // Initialize empty data types _membersKeys = new HashSet <string>(); PointsToMembers = new Dictionary <ulong, KeyValuePair <string, TMember> >(); SortedPoints = new List <ulong>(); // Create universal hash functions using the provided key _baseHashFunction = new UniversalHashFunction(key, maxInputLengthInBytesForUniversalHashFunction); _universalHashFunctionsForEachPoint = new UniversalHashFunction[numberOfPointsOnRingForEachMember]; for (int i = 0; i < _universalHashFunctionsForEachPoint.Length; i++) { _universalHashFunctionsForEachPoint[i] = new UniversalHashFunction(key + ":" + i.ToString(), 16); } // Add the initial members (if their are any) if (initialMembers != null) { AddRange(initialMembers); } }
public void UniversalHashTestBias() { Pseudorandom pseudo = new Pseudorandom(); UniversalHashFunction f = new UniversalHashFunction("Louis Tully as played by Rick Moranis!"); ulong trials = 100000000; ulong[] bitCounts = new ulong[64]; for (ulong trial = 0; trial < trials; trial++) { string randomString = pseudo.GetString(8); UInt64 supposedlyUnbiasedBits = f.Hash(randomString, UniversalHashFunction.MaximumNumberOfResultBitsAllowing32BiasedBits); for (int bit = 0; bit < bitCounts.Length; bit++) { if ((supposedlyUnbiasedBits & (0x8000000000000000ul >> bit)) != 0ul) { bitCounts[bit]++; } } } double[] biases = bitCounts.Select(count => ((0.5d - (((double)count) / ((double)trials)))) / 0.5d).ToArray(); /// The first 32 bits should be unbiased for (int bit = 0; bit < 32; bit++) { double bias = biases[bit]; double biasAbs = Math.Abs(bias); Assert.True(biasAbs < 0.0005d); } }
public DistributedBinomialLadderFilterClient(int numberOfShards, int defaultHeightOfLadder, IDistributedResponsibilitySet <RemoteHost> shardToHostMapping, string configurationKey, TimeSpan?mininmumCacheFreshnessRequired = null) { NumberOfShards = numberOfShards; MaxLadderHeight = defaultHeightOfLadder; MinimumCacheFreshnessRequired = mininmumCacheFreshnessRequired ?? new TimeSpan(0, 0, 1); CacheOfElementsAtTopOfLadder = new FixedSizeLruCache <string, DateTime>(2 * NumberOfShards); ShardHashFunction = new UniversalHashFunction(configurationKey); ShardToHostMapping = shardToHostMapping; }
public MaxWeightHashing(string key, IEnumerable <KeyValuePair <string, TMemberType> > initialMembers = null) { _masterKey = key; _membersAndTheirHashFunctionsAsArray = null; _baseHashFunction = new UniversalHashFunction(key); if (initialMembers != null) { AddRange(initialMembers); } }
public DistributedResponsibilitySet( string configurationKey, int numberOfVirtualNodes, int numberOfPhysicalNodesPerVirtualNode, IList <string> physicalNodes) { ConfigurationKey = configurationKey; KeyHash = new UniversalHashFunction(configurationKey); NumberOfVirtualNodes = numberOfVirtualNodes; NumberOfPhysicalNodesPerVirtualNode = numberOfPhysicalNodesPerVirtualNode; SortedNodeRelationshipScores = new SortedSet <NodeRelationshipScore>(new NodeRelationshipScoreComparer()); PhysicalNodeToNodeRelationships = new Dictionary <string, NodeRelationshipScore[]>(); PhysicalNodes = new HashSet <string>(); Add(physicalNodes); }
public List <TMemberType> FindMembersResponsible(string key, int numberOfUniqueMembersToFind) { TMemberType[] highestScoringMembers = new TMemberType[numberOfUniqueMembersToFind]; ulong[] highestScores = new ulong[numberOfUniqueMembersToFind]; UInt32 intermediateHashValue = (UInt32)_baseHashFunction.Hash(key); MemberAndItsHashFunction[] localMembersAndTheirHashFunctions = MembersAndTheirHashFunctionsAsArray; foreach (MemberAndItsHashFunction memberAndHash in localMembersAndTheirHashFunctions) { TMemberType member = memberAndHash.Member; UniversalHashFunction hashFunction = memberAndHash.HashFunction; ulong score = hashFunction.Hash(intermediateHashValue, UniversalHashFunction.MaximumNumberOfResultBitsAllowing32BiasedBits); int indexToWriteInto; for (indexToWriteInto = numberOfUniqueMembersToFind; indexToWriteInto >= 1 && score > highestScores[indexToWriteInto - 1]; indexToWriteInto--) { } while (indexToWriteInto < numberOfUniqueMembersToFind) { TMemberType evictedMember = highestScoringMembers[indexToWriteInto]; ulong evictedScore = highestScores[indexToWriteInto]; highestScoringMembers[indexToWriteInto] = member; highestScores[indexToWriteInto] = score; indexToWriteInto++; member = evictedMember; score = evictedScore; } } List <TMemberType> result = new List <TMemberType>(numberOfUniqueMembersToFind); for (int i = 0; i < numberOfUniqueMembersToFind && !highestScoringMembers[i].Equals(default(TMemberType)); i++) { result.Add(highestScoringMembers[i]); } return(result); }
protected void AddInsideLock(string physicalNode) { if (!PhysicalNodeToNodeRelationships.ContainsKey(physicalNode)) { NodeRelationshipScore[] nodeRelationshipScores = new NodeRelationshipScore[NumberOfVirtualNodes]; UniversalHashFunction hash = new UniversalHashFunction(ConfigurationKey + physicalNode); for (int i = 0; i < NumberOfVirtualNodes; i++) { nodeRelationshipScores[i].PhysicalNode = physicalNode; nodeRelationshipScores[i].VirtualNode = i; nodeRelationshipScores[i].Score = hash.Hash(i); } } foreach (NodeRelationshipScore nrs in PhysicalNodeToNodeRelationships[physicalNode]) { SortedNodeRelationshipScores.Add(nrs); } PhysicalNodes.Add(physicalNode); }
public ConsistentHashRing(string key, IEnumerable <KeyValuePair <string, TMember> > initialMembers = null, int maxInputLengthInBytesForUniversalHashFunction = DefaultMaxInputLengthInBytesForUniversalHashFunction, int numberOfPointsOnRingForEachMember = DefaultNumberOfPointsOnRingForEachMember) { _numberOfPointsOnRingForEachMember = numberOfPointsOnRingForEachMember; _membersKeys = new HashSet <string>(); PointsToMembers = new Dictionary <ulong, KeyValuePair <string, TMember> >(); SortedPoints = new List <ulong>(); _baseHashFunction = new UniversalHashFunction(key, maxInputLengthInBytesForUniversalHashFunction); _universalHashFunctionsForEachPoint = new UniversalHashFunction[numberOfPointsOnRingForEachMember]; for (int i = 0; i < _universalHashFunctionsForEachPoint.Length; i++) { _universalHashFunctionsForEachPoint[i] = new UniversalHashFunction(key + ":" + i.ToString(), 16); } if (initialMembers != null) { AddRange(initialMembers); } }
public IEnumerable <int> GetIndexesAssociatedWithAnElement(byte[] element, int?numberOfIndexesRequested = null) { HashSet <int> indexesIntoBitArray = new HashSet <int>(); int numberOfBitIndexesToCreate = Math.Min(HashFunctionsMappingElementsToBitsInTheArray.Length, numberOfIndexesRequested ?? int.MaxValue); for (int i = 0; i < numberOfBitIndexesToCreate; i++) { UniversalHashFunction hashFunction = HashFunctionsMappingElementsToBitsInTheArray[i]; byte[] valueToHash = ManagedSHA256.Hash(element); do { int indexIntoBitArray = (int)(hashFunction.Hash(valueToHash) % (uint)Length); if (indexesIntoBitArray.Add(indexIntoBitArray)) { break; } valueToHash = ManagedSHA256.Hash(valueToHash); } while (true); } return(indexesIntoBitArray); }
public FilterArray(int numberOfBitsInArray, int maximumBitIndexesPerElement, bool initilizeBitsOfArrayAtRandom, string saltForHashFunctions = "") { int capacityInBytes = (numberOfBitsInArray + 7) / 8; HashFunctionsMappingElementsToBitsInTheArray = new UniversalHashFunction[maximumBitIndexesPerElement]; for (int i = 0; i < HashFunctionsMappingElementsToBitsInTheArray.Length; i++) { HashFunctionsMappingElementsToBitsInTheArray[i] = new UniversalHashFunction(i + ":" + saltForHashFunctions, 64); } if (initilizeBitsOfArrayAtRandom) { byte[] initialBitValues = new byte[capacityInBytes]; StrongRandomNumberGenerator.GetBytes(initialBitValues); BitArray = new BitArray(initialBitValues); } else { BitArray = new BitArray(capacityInBytes * 8); } }