/// <summary>
/// Map a element to its corresponding bits in the array.
/// </summary>
/// <param name="element">The element to be mapped to indexes</param>
/// <param name="numberOfIndexesRequested">The number of indexes to associate with the element. If not set,
/// uses the MaximumBitIndexesPerElement specified in the constructor.</param>
/// <returns>The array indexes associated with the element.</returns>
public IEnumerable <int> GetIndexesAssociatedWithAnElement(byte[] element, int?numberOfIndexesRequested = null)
{
// Build a set of indexes into the bit array associated with the element
HashSet <int> indexesIntoBitArray = new HashSet <int>();
// Determine how many indexes to generate for this element.
// The lesser of the maximum number supported and the number requested (if provided)
int numberOfBitIndexesToCreate = Math.Min(HashFunctionsMappingElementsToBitsInTheArray.Length, numberOfIndexesRequested ?? int.MaxValue);
// Use one hash function to generate each index
for (int i = 0; i < numberOfBitIndexesToCreate; i++)
{
UniversalHashFunction hashFunction = HashFunctionsMappingElementsToBitsInTheArray[i];
byte[] valueToHash = ManagedSHA256.Hash(element);
do
{
// Use the hash function to index into the array of bits
int indexIntoBitArray = (int)(hashFunction.Hash(valueToHash) % (uint)Length);
if (indexesIntoBitArray.Add(indexIntoBitArray))
{
// Common case: this index points to a bit that is not yet associated with the element
// This index can thus represent the i_th bit for this element.
break;
}
// This hash function generated an index to a bit that is already associated with this element.
// We'll need to rehash to create an index for the i_th bit that was not already assigned.
valueToHash = ManagedSHA256.Hash(valueToHash);
} while (true);
}
return(indexesIntoBitArray);
}
/// <summary>
/// Construct a filter array.
/// </summary>
/// <param name="numberOfBitsInArray">The size of the array in bits.</param>
/// <param name="maximumBitIndexesPerElement">The maximum (and default) number of indexes (bits) in the array to associate with elements.</param>
/// <param name="initilizeBitsOfArrayAtRandom">If set to true, the bits of the filter array will be set to 0 or 1 at random (indpendently, each with probability 0.5).</param>
/// <param name="saltForHashFunctions">A salt used to generate the hash functions.
/// Any two filter arrays generated with the same salt will use the same hash functions.
/// The salt should be kept secret from attackerse who might try to manipulate the selection of elements,
/// such as to intentionally cause bit collisions with the array.</param>
public FilterArray(int numberOfBitsInArray, int maximumBitIndexesPerElement, bool initilizeBitsOfArrayAtRandom,
string saltForHashFunctions = "")
{
// Align on byte boundary to guarantee no less than numberOfBitsInArray
int capacityInBytes = (numberOfBitsInArray + 7) / 8;
// Create hash functions to map elements to indexes in the bit array.
HashFunctionsMappingElementsToBitsInTheArray = new UniversalHashFunction[maximumBitIndexesPerElement];
for (int i = 0; i < HashFunctionsMappingElementsToBitsInTheArray.Length; i++)
{
HashFunctionsMappingElementsToBitsInTheArray[i] =
new UniversalHashFunction(i + ":" + saltForHashFunctions, 64);
}
if (initilizeBitsOfArrayAtRandom)
{
// Initialize the bit array setting ~half the bits randomly to zero by using the
// cryptographic random number generator.
byte[] initialBitValues = new byte[capacityInBytes];
StrongRandomNumberGenerator.GetBytes(initialBitValues);
BitArray = new BitArray(initialBitValues);
}
else
{
// Start with all bits of the array set to zero.
BitArray = new BitArray(capacityInBytes * 8);
}
}
/// <summary>
/// Create a hash ring
/// </summary>
/// <param name="key">A key used for creating hash functions. The key must be kept secret
/// from adversaries that might attempt to perform algorithmic complexity attacks to do such
/// things as ensuring all the important values that one might hash map to the same member.</param>
/// <param name="initialMembers">An optional set of members to place onto the ring.</param>
/// <param name="maxInputLengthInBytesForUniversalHashFunction"></param>
/// <param name="numberOfPointsOnRingForEachMember"></param>
public ConsistentHashRing(string key,
IEnumerable <KeyValuePair <string, TMember> > initialMembers = null,
int maxInputLengthInBytesForUniversalHashFunction = DefaultMaxInputLengthInBytesForUniversalHashFunction,
int numberOfPointsOnRingForEachMember = DefaultNumberOfPointsOnRingForEachMember)
{
_numberOfPointsOnRingForEachMember = numberOfPointsOnRingForEachMember;
// Initialize empty data types
_membersKeys = new HashSet <string>();
PointsToMembers = new Dictionary <ulong, KeyValuePair <string, TMember> >();
SortedPoints = new List <ulong>();
// Create universal hash functions using the provided key
_baseHashFunction = new UniversalHashFunction(key, maxInputLengthInBytesForUniversalHashFunction);
_universalHashFunctionsForEachPoint = new UniversalHashFunction[numberOfPointsOnRingForEachMember];
for (int i = 0; i < _universalHashFunctionsForEachPoint.Length; i++)
{
_universalHashFunctionsForEachPoint[i] = new UniversalHashFunction(key + ":" + i.ToString(), 16);
}
// Add the initial members (if their are any)
if (initialMembers != null)
{
AddRange(initialMembers);
}
}
public void UniversalHashTestBias()
{
Pseudorandom pseudo = new Pseudorandom();
UniversalHashFunction f = new UniversalHashFunction("Louis Tully as played by Rick Moranis!");
ulong trials = 100000000;
ulong[] bitCounts = new ulong[64];
for (ulong trial = 0; trial < trials; trial++)
{
string randomString = pseudo.GetString(8);
UInt64 supposedlyUnbiasedBits = f.Hash(randomString, UniversalHashFunction.MaximumNumberOfResultBitsAllowing32BiasedBits);
for (int bit = 0; bit < bitCounts.Length; bit++)
{
if ((supposedlyUnbiasedBits & (0x8000000000000000ul >> bit)) != 0ul)
{
bitCounts[bit]++;
}
}
}
double[] biases = bitCounts.Select(count => ((0.5d - (((double)count) / ((double)trials)))) / 0.5d).ToArray();
/// The first 32 bits should be unbiased
for (int bit = 0; bit < 32; bit++)
{
double bias = biases[bit];
double biasAbs = Math.Abs(bias);
Assert.True(biasAbs < 0.0005d);
}
}
public DistributedBinomialLadderFilterClient(int numberOfShards, int defaultHeightOfLadder, IDistributedResponsibilitySet <RemoteHost> shardToHostMapping, string configurationKey, TimeSpan?mininmumCacheFreshnessRequired = null)
{
NumberOfShards = numberOfShards;
MaxLadderHeight = defaultHeightOfLadder;
MinimumCacheFreshnessRequired = mininmumCacheFreshnessRequired ?? new TimeSpan(0, 0, 1);
CacheOfElementsAtTopOfLadder = new FixedSizeLruCache <string, DateTime>(2 * NumberOfShards);
ShardHashFunction = new UniversalHashFunction(configurationKey);
ShardToHostMapping = shardToHostMapping;
}
public MaxWeightHashing(string key,
IEnumerable <KeyValuePair <string, TMemberType> > initialMembers = null)
{
_masterKey = key;
_membersAndTheirHashFunctionsAsArray = null;
_baseHashFunction = new UniversalHashFunction(key);
if (initialMembers != null)
{
AddRange(initialMembers);
}
}
public DistributedResponsibilitySet(
string configurationKey,
int numberOfVirtualNodes,
int numberOfPhysicalNodesPerVirtualNode,
IList <string> physicalNodes)
{
ConfigurationKey = configurationKey;
KeyHash = new UniversalHashFunction(configurationKey);
NumberOfVirtualNodes = numberOfVirtualNodes;
NumberOfPhysicalNodesPerVirtualNode = numberOfPhysicalNodesPerVirtualNode;
SortedNodeRelationshipScores = new SortedSet <NodeRelationshipScore>(new NodeRelationshipScoreComparer());
PhysicalNodeToNodeRelationships = new Dictionary <string, NodeRelationshipScore[]>();
PhysicalNodes = new HashSet <string>();
Add(physicalNodes);
}
public List <TMemberType> FindMembersResponsible(string key, int numberOfUniqueMembersToFind)
{
TMemberType[] highestScoringMembers = new TMemberType[numberOfUniqueMembersToFind];
ulong[] highestScores = new ulong[numberOfUniqueMembersToFind];
UInt32 intermediateHashValue = (UInt32)_baseHashFunction.Hash(key);
MemberAndItsHashFunction[] localMembersAndTheirHashFunctions =
MembersAndTheirHashFunctionsAsArray;
foreach (MemberAndItsHashFunction memberAndHash in localMembersAndTheirHashFunctions)
{
TMemberType member = memberAndHash.Member;
UniversalHashFunction hashFunction = memberAndHash.HashFunction;
ulong score = hashFunction.Hash(intermediateHashValue, UniversalHashFunction.MaximumNumberOfResultBitsAllowing32BiasedBits);
int indexToWriteInto;
for (indexToWriteInto = numberOfUniqueMembersToFind;
indexToWriteInto >= 1 && score > highestScores[indexToWriteInto - 1];
indexToWriteInto--)
{
}
while (indexToWriteInto < numberOfUniqueMembersToFind)
{
TMemberType evictedMember = highestScoringMembers[indexToWriteInto];
ulong evictedScore = highestScores[indexToWriteInto];
highestScoringMembers[indexToWriteInto] = member;
highestScores[indexToWriteInto] = score;
indexToWriteInto++;
member = evictedMember;
score = evictedScore;
}
}
List <TMemberType> result = new List <TMemberType>(numberOfUniqueMembersToFind);
for (int i = 0; i < numberOfUniqueMembersToFind && !highestScoringMembers[i].Equals(default(TMemberType)); i++)
{
result.Add(highestScoringMembers[i]);
}
return(result);
}
protected void AddInsideLock(string physicalNode)
{
if (!PhysicalNodeToNodeRelationships.ContainsKey(physicalNode))
{
NodeRelationshipScore[] nodeRelationshipScores = new NodeRelationshipScore[NumberOfVirtualNodes];
UniversalHashFunction hash = new UniversalHashFunction(ConfigurationKey + physicalNode);
for (int i = 0; i < NumberOfVirtualNodes; i++)
{
nodeRelationshipScores[i].PhysicalNode = physicalNode;
nodeRelationshipScores[i].VirtualNode = i;
nodeRelationshipScores[i].Score = hash.Hash(i);
}
}
foreach (NodeRelationshipScore nrs in PhysicalNodeToNodeRelationships[physicalNode])
{
SortedNodeRelationshipScores.Add(nrs);
}
PhysicalNodes.Add(physicalNode);
}
public ConsistentHashRing(string key,
IEnumerable <KeyValuePair <string, TMember> > initialMembers = null,
int maxInputLengthInBytesForUniversalHashFunction = DefaultMaxInputLengthInBytesForUniversalHashFunction,
int numberOfPointsOnRingForEachMember = DefaultNumberOfPointsOnRingForEachMember)
{
_numberOfPointsOnRingForEachMember = numberOfPointsOnRingForEachMember;
_membersKeys = new HashSet <string>();
PointsToMembers = new Dictionary <ulong, KeyValuePair <string, TMember> >();
SortedPoints = new List <ulong>();
_baseHashFunction = new UniversalHashFunction(key, maxInputLengthInBytesForUniversalHashFunction);
_universalHashFunctionsForEachPoint = new UniversalHashFunction[numberOfPointsOnRingForEachMember];
for (int i = 0; i < _universalHashFunctionsForEachPoint.Length; i++)
{
_universalHashFunctionsForEachPoint[i] = new UniversalHashFunction(key + ":" + i.ToString(), 16);
}
if (initialMembers != null)
{
AddRange(initialMembers);
}
}
public IEnumerable <int> GetIndexesAssociatedWithAnElement(byte[] element, int?numberOfIndexesRequested = null)
{
HashSet <int> indexesIntoBitArray = new HashSet <int>();
int numberOfBitIndexesToCreate = Math.Min(HashFunctionsMappingElementsToBitsInTheArray.Length, numberOfIndexesRequested ?? int.MaxValue);
for (int i = 0; i < numberOfBitIndexesToCreate; i++)
{
UniversalHashFunction hashFunction = HashFunctionsMappingElementsToBitsInTheArray[i];
byte[] valueToHash = ManagedSHA256.Hash(element);
do
{
int indexIntoBitArray = (int)(hashFunction.Hash(valueToHash) % (uint)Length);
if (indexesIntoBitArray.Add(indexIntoBitArray))
{
break;
}
valueToHash = ManagedSHA256.Hash(valueToHash);
} while (true);
}
return(indexesIntoBitArray);
}
public FilterArray(int numberOfBitsInArray, int maximumBitIndexesPerElement, bool initilizeBitsOfArrayAtRandom,
string saltForHashFunctions = "")
{
int capacityInBytes = (numberOfBitsInArray + 7) / 8;
HashFunctionsMappingElementsToBitsInTheArray = new UniversalHashFunction[maximumBitIndexesPerElement];
for (int i = 0; i < HashFunctionsMappingElementsToBitsInTheArray.Length; i++)
{
HashFunctionsMappingElementsToBitsInTheArray[i] =
new UniversalHashFunction(i + ":" + saltForHashFunctions, 64);
}
if (initilizeBitsOfArrayAtRandom)
{
byte[] initialBitValues = new byte[capacityInBytes];
StrongRandomNumberGenerator.GetBytes(initialBitValues);
BitArray = new BitArray(initialBitValues);
}
else
{
BitArray = new BitArray(capacityInBytes * 8);
}
}
