static void RunAll <TKey>(List <TKey> keys, string content)
{
var xxHash = XXHash32.Create();
xxHash.Initialize();
var murmur = new System.Data.HashFunction.MurmurHash3();
var city = new System.Data.HashFunction.CityHash();
// Default:
DrawHashes(keys, x => x.GetHashCode(), $"hash_{content}_default");
// xxHash:
DrawHashes(keys, x =>
{
var h = xxHash.ComputeHash(ToBytes((dynamic)x));
return(BitConverter.ToInt32(h, 0));
},
$"hash_{content}_xx");
// Murmur:
DrawHashes(keys, x =>
{
var h = murmur.ComputeHash(ToBytes((dynamic)x));
return(BitConverter.ToInt32(h, 0));
},
$"hash_{content}_murmur");
// City:
DrawHashes(keys, x =>
{
var h = city.ComputeHash(ToBytes((dynamic)x));
return(BitConverter.ToInt32(h, 0));
},
$"hash_{content}_city");
}
public HashBenchmark()
{
this.data = new byte[N];
new Random(42).NextBytes(this.data);
this.farm = new FarmHash();
this.xxh32 = new XXHash32();
this.xxh64 = new XxHash64();
}
public void SingleBlockXxh32UsingSpanMatchesTheirs(string text)
{
var input = System.Text.Encoding.UTF8.GetBytes(text);
var expected = Theirs32(input);
var actual = XXHash32.DigestOf(input.AsSpan());
Assert.Equal(expected, actual);
}
/// <summary>
/// Hashes given partition and key.
/// </summary>
/// <param name="partition">Partition.</param>
/// <param name="key">Key.</param>
/// <returns>Long hash for given partition and key.</returns>
public static long HashPartitionAndKey(string partition, string key)
{
unchecked
{
var pHash = (long)XXHash32.CalculateRaw(partition) << 32;
return(pHash + XXHash32.CalculateRaw(key));
}
}
public static unsafe (List <Facet> Facets, long FacetsEtag) ParseFromJson(BlittableJsonReaderArray array)
{
var results = new List <Facet>();
foreach (BlittableJsonReaderObject facetAsJson in array)
{
results.Add(JsonDeserializationServer.Facet(facetAsJson));
}
return(results, Hashing.XXHash32.Calculate(array.Parent.BasePointer, array.Parent.Size));
}
/// <summary>
/// Hashes given partition; returns null if partition is null.
/// </summary>
/// <param name="partition">Partition.</param>
/// <returns>Long hash for given partition, null if partition is null.</returns>
public static long?HashPartition(string partition)
{
if (partition == null)
{
return(new long?());
}
unchecked
{
return((long)XXHash32.CalculateRaw(partition) << 32);
}
}
public void HashAlgorithmWrapperReturnsSameResults(int seed, int length)
{
var bytes = new byte[length];
new Random(seed).NextBytes(bytes);
var expected = XXHash32.DigestOf(bytes, 0, bytes.Length);
//var actual = new XXHash32().AsHashAlgorithm().ComputeHash(bytes);
var actual = ((HashAlgorithm) new XXHash32()).ComputeHash(bytes);
Assert.Equal(expected, BitConverter.ToUInt32(actual, 0));
}
private void ReadFrame()
{
Read0();
uint? magic = TryRead32();
if (magic != 0x184D2204)
throw new InvalidDataException(RS.ExpectLZ4MagicNumber);
Read0();
ushort flgBd = Read16();
int flg = flgBd & 0xFF;
int bd = (flgBd >> 8) & 0xFF;
int version = (flg >> 6) & 0x11;
if (version != 1)
throw new InvalidDataException(string.Format(RS.LZ4VersionNotSupported, version));
bool blockChaining = ((flg >> 5) & 0x01) == 0;
bool blockChecksum = ((flg >> 4) & 0x01) != 0;
bool hasContentSize = ((flg >> 3) & 0x01) != 0;
bool contentChecksum = ((flg >> 2) & 0x01) != 0;
bool hasDictionary = (flg & 0x01) != 0;
int blockSizeCode = (bd >> 4) & 0x07;
long? contentLength = hasContentSize ? (long?) Read64() : null;
uint? dictionaryId = hasDictionary ? (uint?) Read32() : null;
byte actualHC = (byte)(XXHash32.DigestOf(_buffer16, 0, _index16) >> 8);
byte expectedHC = Read8();
if (actualHC != expectedHC)
throw new InvalidDataException(RS.BadLZ4FrameHeaderChecksum);
int blockSize = MaxBlockSize(blockSizeCode);
if (hasDictionary)
{
// Write32(dictionaryId);
throw new NotImplementedException(string.Format(RS.FeatureNotImplementedInType, "Predefined Dictionaries", GetType().Name));
}
_frameInfo = new LZ4FrameDescriptor(contentLength, contentChecksum, blockChaining, blockChecksum, dictionaryId, blockSize);
_decoder = _decoderFactory(_frameInfo);
_buffer = new byte[blockSize];
}
private void WriteFrame()
{
Write32(0x184D2204);
Flush16();
const int versionCode = 0x01;
bool blockChaining = _descriptor.Chaining;
bool blockChecksum = _descriptor.BlockChecksum;
bool contentChecksum = _descriptor.ContentChecksum;
bool hasContentSize = _descriptor.ContentLength.HasValue;
bool hasDictionary = _descriptor.Dictionary.HasValue;
int flg = (versionCode << 6) |
((blockChaining ? 0 : 1) << 5) |
((blockChecksum ? 1 : 0) << 4) |
((hasContentSize ? 1 : 0) << 3) |
((contentChecksum ? 1 : 0) << 2) |
(hasDictionary ? 1 : 0);
int blockSize = _descriptor.BlockSize;
int bd = MaxBlockSizeCode(blockSize) << 4;
Write16((ushort) ((flg & 0xFF) | (bd & 0xFF) << 8));
if (hasContentSize)
{
// Write64(contentSize)
throw new NotImplementedException(string.Format(RS.FeatureNotImplementedInType, "Content Size", GetType().Name));
}
if (hasDictionary)
{
// Write32(dictionaryId)
throw new NotImplementedException(string.Format(RS.FeatureNotImplementedInType, "Predefined Dictionaries", GetType().Name));
}
byte hc = (byte)(XXHash32.DigestOf(_buffer16, 0, _index16) >> 8);
Write8(hc);
Flush16();
_encoder = CreateEncoder();
_buffer = new byte[LZ4Codec.MaximumOutputSize(blockSize)];
}
public void EveryCallToDigestReturnsSameHash(int seed, int length)
{
var random = new Random(seed);
var bytes = new byte[length];
random.NextBytes(bytes);
var expected = XXHash32.DigestOf(bytes, 0, bytes.Length);
var transform = new XXHash32();
transform.Update(bytes, 0, length);
for (var i = 0; i < 100; i++)
{
Assert.Equal(expected, transform.Digest());
}
}
public override int GetHashCode()
{
var hash = new XXHash32();
hash.Update(ToBytes(RequestUri.ToString()));
hash.Update(ToBytes(Method.Method));
foreach (var kv in Headers)
{
// Case-insensitive keys
hash.Update(ToBytes(kv.Key.ToLowerInvariant()));
foreach (var headerValue in kv.Value)
{
// Case-sensitive values?
hash.Update(ToBytes(headerValue));
}
}
return((int)hash.Result);
}
public unsafe void EmptyHash()
{
var input = Array.Empty <byte>();
var expected = Theirs32(input);
var actual1 = XXHash32.DigestOf(input, 0, input.Length);
Assert.Equal(expected, actual1);
fixed(byte *inputPtr = input)
{
var actual2 = XXHash32.DigestOf(inputPtr, 0);
Assert.Equal(expected, actual2);
}
var actual3 = XXHash32.EmptyHash;
Assert.Equal(expected, actual3);
}
public void RestartableHashReturnsSameResultsAsSingleBlock(int seed, int length, int chunk)
{
var random = new Random(seed);
var bytes = new byte[length];
random.NextBytes(bytes);
var expected = XXHash32.DigestOf(bytes, 0, bytes.Length);
var transform = new XXHash32();
var index = 0;
while (index < length)
{
var l = Math.Min(chunk, length - index);
transform.Update(bytes, index, l);
index += l;
}
var actual = transform.Digest();
Assert.Equal(expected, actual);
}
public uint xxHash32_string_hash() => XXHash32.Hash(testString);
public uint xxHash32_binary_hash() => XXHash32.Hash(testBinary);
public void TestHashUpdate()
{
const int N = 1_000_000;
var random = new Random(42);
var data = new byte[N];
random.NextBytes(data);
// CRC-32
var crc32 = new Crc32();
for (var n = 0; n < 1000; n++)
{
var span = data.AsSpan(0, n);
var h = BitConverter.ToUInt32(crc32.GetHash(span));
var h2 = Crc32.Hash32(span);
Assert.Equal(h, h2);
}
this.TestHashUpdate_do(crc32, data, random);
// Adler-32
var adler32 = new Adler32();
for (var n = 0; n < 1000; n++)
{
var span = data.AsSpan(0, n);
var h = BitConverter.ToUInt32(adler32.GetHash(span));
var h2 = Adler32.Hash32(span);
Assert.Equal(h, h2);
}
this.TestHashUpdate_do(adler32, data, random);
// FarmHash
var farm = new FarmHash();
for (var n = 0; n < 1000; n++)
{
var span = data.AsSpan(0, n);
var h = BitConverter.ToUInt64(farm.GetHash(span));
var h2 = FarmHash.Hash64(span);
Assert.Equal(h, h2);
}
this.TestHashUpdate_do(farm, data, random);
// xxHash32
var xxh32 = new XXHash32();
for (var n = 0; n < 1000; n++)
{
var span = data.AsSpan(0, n);
var h = BitConverter.ToUInt32(xxh32.GetHash(span));
var h2 = XXHash32.Hash32(span);
Assert.Equal(h, h2);
}
this.TestHashUpdate_do(xxh32, data, random);
// xxHash64
var xxh64 = new XxHash64();
for (var n = 0; n < 1000; n++)
{
var span = data.AsSpan(0, n);
var h = BitConverter.ToUInt64(xxh64.GetHash(span));
var h2 = XxHash64.Hash64(span);
Assert.Equal(h, h2);
}
this.TestHashUpdate_do(xxh64, data, random);
// Sha1
using var sha1 = new Arc.Crypto.Sha1();
this.TestHashUpdate_do(sha1, data, random);
// Sha2_256
using var sha2_256 = new Arc.Crypto.Sha2_256();
this.TestHashUpdate_do(sha2_256, data, random);
// Sha2_384
using var sha2_384 = new Arc.Crypto.Sha2_384();
this.TestHashUpdate_do(sha2_384, data, random);
// Sha2_512
using var sha2_512 = new Arc.Crypto.Sha2_512();
this.TestHashUpdate_do(sha2_512, data, random);
// Sha3_256
var sha3_256 = new Arc.Crypto.Sha3_256();
this.TestHashUpdate_do(sha3_256, data, random);
// Sha3_384
var sha3_384 = new Arc.Crypto.Sha3_384();
this.TestHashUpdate_do(sha3_384, data, random);
// Sha3_512
var sha3_512 = new Arc.Crypto.Sha3_512();
this.TestHashUpdate_do(sha3_512, data, random);
}
public static uint CalculateXXHash32(byte[] buf, int len = -1, uint seed = 0)
{
uint hash = XXHash32.CalculateHash(buf, len, seed);
return(hash);
}
