private bool AddKeyValue(byte[] key, TValue value) { // lock(cs) required. var table = this.hashTable; var hash = unchecked ((int)FarmHash.Hash64(key)); var h = hash & (table.Length - 1); if (table[h] == null) { var item = new Item(key, value, hash); System.Threading.Volatile.Write(ref table[h], item); } else { var i = table[h] !; while (true) { if (key.SequenceEqual(i.Key) == true) { // Identical return(false); } if (i.Next == null) { // Last item. break; } i = i.Next; } var item = new Item(key, value, hash); System.Threading.Volatile.Write(ref i.Next, item); } return(true); }
private static unsafe ulong HashLen0to16(byte *data, int length) { if (length >= 8) { ulong mul = FarmHashConstants.k2 + (uint)length * 2; ulong a = Utilities.Fetch64(data) + FarmHashConstants.k2; ulong b = Utilities.Fetch64(data, length - 8); ulong c = Utilities.RotateWithCheck(b, 37) * mul + a; ulong d = (Utilities.RotateWithCheck(a, 25) + b) * mul; return(HashLen16(c, d, mul)); } if (length >= 4) { ulong mul = FarmHashConstants.k2 + (uint)length * 2; ulong a = Utilities.Fetch32(data, 0); return(HashLen16((uint)length + (a << 3), Utilities.Fetch32(data, length - 4), mul)); } if (length > 0) { byte a = data[0]; byte b = data[length >> 1]; byte c = data[length - 1]; uint y = a + ((uint)b << 8); uint z = (uint)length + ((uint)c << 2); return(FarmHash.ShiftMix(y * FarmHashConstants.k2 ^ z * FarmHashConstants.k0) * FarmHashConstants.k2); } return(FarmHashConstants.k2); }
public HashBenchmark() { this.data = new byte[N]; new Random(42).NextBytes(this.data); this.farm = new FarmHash(); this.xxh32 = new XXHash32(); this.xxh64 = new XxHash64(); }
private static unsafe uint Hash32Len5to12(byte *s, uint len, uint seed = 0) { uint a = len, b = len * 5, c = 9, d = b + seed; a += Utilities.Fetch32(s); b += Utilities.Fetch32(s + len - 4); c += Utilities.Fetch32(s + ((len >> 1) & 4)); return(Utilities.FMix(seed ^ FarmHash.Mur(c, FarmHash.Mur(b, FarmHash.Mur(a, d))))); }
public static int GetHashCode(byte[] bytes, int offset, int count) { if (Is32Bit) { return(unchecked ((int)FarmHash.Hash32(bytes, offset, count))); } else { return(unchecked ((int)FarmHash.Hash64(bytes, offset, count))); } }
private static unsafe uint Hash32Len0to4(byte *s, uint len, uint seed = 0) { uint b = seed; uint c = 9; for (int i = 0; i < len; i++) { b = b * FarmHashConstants.c1 + *(s + i); c ^= b; } return(Utilities.FMix(FarmHash.Mur(b, FarmHash.Mur(len, c)))); }
private static unsafe ulong H32(byte *data, int offset, int length, ulong mul, ulong seed0 = 0, ulong seed1 = 0) { ulong a = Utilities.Fetch64(data, offset) * FarmHashConstants.k1; ulong b = Utilities.Fetch64(data, 8 + offset); ulong c = Utilities.Fetch64(data, length - 8 + offset) * mul; ulong d = Utilities.Fetch64(data, length - 16 + offset) * FarmHashConstants.k2; ulong u = Utilities.RotateWithCheck(a + b, 43) + Utilities.RotateWithCheck(c, 30) + d + seed0; ulong v = a + Utilities.RotateWithCheck(b + FarmHashConstants.k2, 18) + c + seed1; a = FarmHash.ShiftMix((u ^ v) * mul); b = FarmHash.ShiftMix((v ^ a) * mul); return(b); }
private static uint Hash32Len0to4(byte[] s, uint len, uint seed = 0) { uint b = seed; uint c = 9; for (int i = 0; i < len; i++) { byte v = s[i]; b = b * FarmHashConstants.c1 + v; c ^= b; } return(Utilities.FMix(FarmHash.Mur(b, FarmHash.Mur(len, c)))); }
/// <summary> /// Decrypts data using the specified password. /// </summary> /// <param name="encrypted">The encrypted data.</param> /// <param name="password">The password.</param> /// <param name="data">The decrypted data.</param> /// <returns><see langword="true"/> if the decryption was successful; otherwise, <see langword="false"/>.</returns> public static bool TryDecrypt(ReadOnlySpan <byte> encrypted, ReadOnlySpan <byte> password, out Memory <byte> data) { data = default; if (encrypted.Length < SaltLength) { return(false); } // Hash: Sha3_384 => Key(32) + IV(16) var keyIV = GetKeyIV(encrypted.Slice(0, SaltLength), password); // AES byte[] decrypted; using (var aes = Aes.Create()) { aes.Key = keyIV.Slice(0, aes.KeySize / 8).ToArray(); if (keyIV.Length != ((aes.KeySize / 8) + (aes.BlockSize / 8))) { throw new InvalidOperationException(); } // Salt[8], Encrypted[8 + 8 + DataLength] (Random[8], Checksum[8 = FarmHash64], Data[DataLength]) try { decrypted = aes.DecryptCbc(encrypted.Slice(SaltLength), keyIV.Slice(aes.KeySize / 8), DefaultPaddingMode); } catch { return(false); } } var dataPosition = RandomLength + ChecksumLength; if (decrypted.Length < dataPosition) { return(false); } // Checksum: FarmHash64 var checksum = FarmHash.Hash64(decrypted.AsSpan(dataPosition)); if (BitConverter.ToUInt64(decrypted.AsSpan(RandomLength)) != checksum) { return(false); } data = decrypted.AsMemory(dataPosition); return(true); }
private static unsafe uint Hash32Len13to24(byte *s, uint len, uint seed = 0) { uint a = Utilities.Fetch32(s - 4 + (len >> 1)); uint b = Utilities.Fetch32(s + 4); uint c = Utilities.Fetch32(s + len - 8); uint d = Utilities.Fetch32(s + (len >> 1)); uint e = Utilities.Fetch32(s); uint f = Utilities.Fetch32(s + len - 4); uint h = d * FarmHashConstants.c1 + len + seed; a = Utilities.RotateWithCheck(a, 12) + f; h = FarmHash.Mur(c, h) + a; a = Utilities.RotateWithCheck(a, 3) + c; h = FarmHash.Mur(e, h) + a; a = Utilities.RotateWithCheck(a + f, 12) + d; h = FarmHash.Mur(b ^ seed, h) + a; return(Utilities.FMix(h)); }
/// <summary> /// Encrypts data using the specified password. /// </summary> /// <param name="data">The data to encrypt.</param> /// <param name="password">The password.</param> /// <returns>The encrypted data.</returns> public static byte[] Encrypt(ReadOnlySpan <byte> data, ReadOnlySpan <byte> password) { // Salt: Random[SaltLength], Random: Random[RandomLength] var randomBuffer = RandomNumberGenerator.GetBytes(SaltLength + RandomLength); var salt = randomBuffer.AsSpan(0, SaltLength); var random = randomBuffer.AsSpan(SaltLength, RandomLength); // Hash: Sha3_384 => Key(32) + IV(16) var keyIV = GetKeyIV(salt, password); // Checksum: FarmHash64 var checksum = FarmHash.Hash64(data); // AES byte[] buffer; using (var aes = Aes.Create()) { aes.Key = keyIV.Slice(0, aes.KeySize / 8).ToArray(); var plainLength = RandomLength + ChecksumLength + data.Length; var cipherLength = aes.GetCiphertextLengthCbc(plainLength, DefaultPaddingMode); if (keyIV.Length != ((aes.KeySize / 8) + (aes.BlockSize / 8))) { throw new InvalidOperationException(); } // Salt[8], Encrypted[8 + 8 + DataLength] (Random[8], Checksum[8 = FarmHash64], Data[DataLength]) var bufferLength = SaltLength + cipherLength; buffer = new byte[bufferLength]; var bufferSpan = buffer.AsSpan(); salt.CopyTo(bufferSpan); bufferSpan = bufferSpan.Slice(SaltLength); random.CopyTo(bufferSpan); bufferSpan = bufferSpan.Slice(RandomLength); BitConverter.TryWriteBytes(bufferSpan, checksum); bufferSpan = bufferSpan.Slice(ChecksumLength); data.CopyTo(bufferSpan); // Encrypt var written = aes.EncryptCbc(buffer.AsSpan(SaltLength, plainLength), keyIV.Slice(aes.KeySize / 8), buffer.AsSpan(SaltLength), DefaultPaddingMode); Debug.Assert(written == cipherLength, "Encrypted length mismatch."); } return(buffer); }
public bool TryGetValue(ReadOnlySpan <byte> key, [MaybeNullWhen(false)] out TValue value) { var table = this.hashTable; var hash = unchecked ((int)FarmHash.Hash64(key)); var item = table[hash & (table.Length - 1)]; while (item != null) { if (key.SequenceEqual(item.Key) == true) { // Identical. alternative: (key == item.Key). value = item.Value; return(true); } item = item.Next; } value = default; return(false); }
private static unsafe ulong Hash64(byte *s, int len) { const ulong seed = 81; if (len <= 32) { if (len <= 16) { return(HashLen0to16(s, len)); } return(HashLen17to32(s, len)); } if (len <= 64) { return(HashLen33to64(s, len)); } // For strings over 64 bytes we loop. Internal state consists of 56 bytes: v, w, x, y, and z. ulong x = seed; ulong y = unchecked (seed * FarmHashConstants.k1) + 113; ulong z = FarmHash.ShiftMix(y * FarmHashConstants.k2 + 113) * FarmHashConstants.k2; Uint128 v = new Uint128(0, 0); Uint128 w = new Uint128(0, 0); x = x * FarmHashConstants.k2 + Utilities.Fetch64(s); // Set end so that after the loop we have 1 to 64 bytes left to process. int index = 0; int end = (len - 1) / 64 * 64; int last64 = end + ((len - 1) & 63) - 63; do { x = Utilities.RotateWithCheck(x + y + v.Low + Utilities.Fetch64(s, 8), 37) * FarmHashConstants.k1; y = Utilities.RotateWithCheck(y + v.High + Utilities.Fetch64(s, 48), 42) * FarmHashConstants.k1; x ^= w.High; y += v.Low + Utilities.Fetch64(s, 40); z = Utilities.RotateWithCheck(z + w.Low, 33) * FarmHashConstants.k1; v = WeakHashLen32WithSeeds(s, 0, v.High * FarmHashConstants.k1, x + w.Low); w = WeakHashLen32WithSeeds(s, 32, z + w.High, y + Utilities.Fetch64(s, 16)); Utilities.Swap(ref z, ref x); index += 64; } while (index != end); ulong mul = FarmHashConstants.k1 + ((z & 0xff) << 1); // Make s point to the last 64 bytes of input. index = last64; w.Low += ((uint)len - 1) & 63; v.Low += w.Low; w.Low += v.Low; x = Utilities.RotateWithCheck(x + y + v.Low + Utilities.Fetch64(s, index + 8), 37) * mul; y = Utilities.RotateWithCheck(y + v.High + Utilities.Fetch64(s, index + 48), 42) * mul; x ^= w.High * 9; y += v.Low * 9 + Utilities.Fetch64(s, index + 40); z = Utilities.RotateWithCheck(z + w.Low, 33) * mul; v = WeakHashLen32WithSeeds(s, index + 0, v.High * mul, x + w.Low); w = WeakHashLen32WithSeeds(s, index + 32, z + w.High, y + Utilities.Fetch64(s, index + 16)); Utilities.Swap(ref z, ref x); return(HashLen16(HashLen16(v.Low, w.Low, mul) + FarmHash.ShiftMix(y) * FarmHashConstants.k0 + z, HashLen16(v.High, w.High, mul) + x, mul)); }
private static unsafe ulong Hash64WithSeeds(byte *s, int len, ulong seed0, ulong seed1) { if (len <= 64) { return(HashLen16(Hash64(s, len) - seed0, seed1, 0x9ddfea08eb382d69UL)); //PORT NOTE: This used to refer to Hash128to64, which was the same as HashLen16, just with hardcoded mul } // For strings over 64 bytes we loop. Internal state consists of // 64 bytes: u, v, w, x, y, and z. ulong x = seed0; ulong y = seed1 * FarmHashConstants.k2 + 113; ulong z = FarmHash.ShiftMix(y * FarmHashConstants.k2) * FarmHashConstants.k2; Uint128 v = new Uint128(seed0, seed1); Uint128 w = new Uint128(0, 0); ulong u = x - z; x *= FarmHashConstants.k2; ulong mul = FarmHashConstants.k2 + (u & 0x82); // Set end so that after the loop we have 1 to 64 bytes left to process. int index = 0; int end = (len - 1) / 64 * 64; int last64 = end + ((len - 1) & 63) - 63; do { ulong a0 = Utilities.Fetch64(s); ulong a1 = Utilities.Fetch64(s, 8); ulong a2 = Utilities.Fetch64(s, 16); ulong a3 = Utilities.Fetch64(s, 24); ulong a4 = Utilities.Fetch64(s, 32); ulong a5 = Utilities.Fetch64(s, 40); ulong a6 = Utilities.Fetch64(s, 48); ulong a7 = Utilities.Fetch64(s, 56); x += a0 + a1; y += a2; z += a3; v.Low += a4; v.High += a5 + a1; w.Low += a6; w.High += a7; x = Utilities.RotateWithCheck(x, 26); x *= 9; y = Utilities.RotateWithCheck(y, 29); z *= mul; v.Low = Utilities.RotateWithCheck(v.Low, 33); v.High = Utilities.RotateWithCheck(v.High, 30); w.Low ^= x; w.Low *= 9; z = Utilities.RotateWithCheck(z, 32); z += w.High; w.High += z; z *= 9; Utilities.Swap(ref u, ref y); z += a0 + a6; v.Low += a2; v.High += a3; w.Low += a4; w.High += a5 + a6; x += a1; y += a7; y += v.Low; v.Low += x - y; v.High += w.Low; w.Low += v.High; w.High += x - y; x += w.High; w.High = Utilities.RotateWithCheck(w.High, 34); Utilities.Swap(ref u, ref z); index += 64; } while (index != end); // Make s point to the last 64 bytes of input. index = last64; u *= 9; v.High = Utilities.RotateWithCheck(v.High, 28); v.Low = Utilities.RotateWithCheck(v.Low, 20); w.Low += ((uint)len - 1) & 63; u += y; y += u; x = Utilities.RotateWithCheck(y - x + v.Low + Utilities.Fetch64(s, index + 8), 37) * mul; y = Utilities.RotateWithCheck(y ^ v.High ^ Utilities.Fetch64(s, index + 48), 42) * mul; x ^= w.High * 9; y += v.Low + Utilities.Fetch64(s, index + 40); z = Utilities.RotateWithCheck(z + w.Low, 33) * mul; v = WeakHashLen32WithSeeds(s, index + 0, v.High * mul, x + w.Low); w = WeakHashLen32WithSeeds(s, index + 32, z + w.High, y + Utilities.Fetch64(s, index + 16)); return(H(HashLen16(v.Low + x, w.Low ^ y, mul) + z - u, H(v.High + y, w.High + z, FarmHashConstants.k2, 30) ^ x, FarmHashConstants.k2, 31)); }
public void TestHashUpdate() { const int N = 1_000_000; var random = new Random(42); var data = new byte[N]; random.NextBytes(data); // CRC-32 var crc32 = new Crc32(); for (var n = 0; n < 1000; n++) { var span = data.AsSpan(0, n); var h = BitConverter.ToUInt32(crc32.GetHash(span)); var h2 = Crc32.Hash32(span); Assert.Equal(h, h2); } this.TestHashUpdate_do(crc32, data, random); // Adler-32 var adler32 = new Adler32(); for (var n = 0; n < 1000; n++) { var span = data.AsSpan(0, n); var h = BitConverter.ToUInt32(adler32.GetHash(span)); var h2 = Adler32.Hash32(span); Assert.Equal(h, h2); } this.TestHashUpdate_do(adler32, data, random); // FarmHash var farm = new FarmHash(); for (var n = 0; n < 1000; n++) { var span = data.AsSpan(0, n); var h = BitConverter.ToUInt64(farm.GetHash(span)); var h2 = FarmHash.Hash64(span); Assert.Equal(h, h2); } this.TestHashUpdate_do(farm, data, random); // xxHash32 var xxh32 = new XXHash32(); for (var n = 0; n < 1000; n++) { var span = data.AsSpan(0, n); var h = BitConverter.ToUInt32(xxh32.GetHash(span)); var h2 = XXHash32.Hash32(span); Assert.Equal(h, h2); } this.TestHashUpdate_do(xxh32, data, random); // xxHash64 var xxh64 = new XxHash64(); for (var n = 0; n < 1000; n++) { var span = data.AsSpan(0, n); var h = BitConverter.ToUInt64(xxh64.GetHash(span)); var h2 = XxHash64.Hash64(span); Assert.Equal(h, h2); } this.TestHashUpdate_do(xxh64, data, random); // Sha1 using var sha1 = new Arc.Crypto.Sha1(); this.TestHashUpdate_do(sha1, data, random); // Sha2_256 using var sha2_256 = new Arc.Crypto.Sha2_256(); this.TestHashUpdate_do(sha2_256, data, random); // Sha2_384 using var sha2_384 = new Arc.Crypto.Sha2_384(); this.TestHashUpdate_do(sha2_384, data, random); // Sha2_512 using var sha2_512 = new Arc.Crypto.Sha2_512(); this.TestHashUpdate_do(sha2_512, data, random); // Sha3_256 var sha3_256 = new Arc.Crypto.Sha3_256(); this.TestHashUpdate_do(sha3_256, data, random); // Sha3_384 var sha3_384 = new Arc.Crypto.Sha3_384(); this.TestHashUpdate_do(sha3_384, data, random); // Sha3_512 var sha3_512 = new Arc.Crypto.Sha3_512(); this.TestHashUpdate_do(sha3_512, data, random); }
public static unsafe uint ComputeHash(byte *s, int len) { if (len <= 4) { return(Hash32Len0to4(s, (uint)len)); } if (len <= 24) { return(len <= 12 ? Hash32Len5to12(s, (uint)len) : Hash32Len13to24(s, (uint)len)); } uint h = (uint)len, g = FarmHashConstants.c1 * (uint)len, f = g; uint a0 = Utilities.RotateWithCheck(Utilities.Fetch32(s + len - 4) * FarmHashConstants.c1, 17) * FarmHashConstants.c2; uint a1 = Utilities.RotateWithCheck(Utilities.Fetch32(s + len - 8) * FarmHashConstants.c1, 17) * FarmHashConstants.c2; uint a2 = Utilities.RotateWithCheck(Utilities.Fetch32(s + len - 16) * FarmHashConstants.c1, 17) * FarmHashConstants.c2; uint a3 = Utilities.RotateWithCheck(Utilities.Fetch32(s + len - 12) * FarmHashConstants.c1, 17) * FarmHashConstants.c2; uint a4 = Utilities.RotateWithCheck(Utilities.Fetch32(s + len - 20) * FarmHashConstants.c1, 17) * FarmHashConstants.c2; h ^= a0; h = Utilities.RotateWithCheck(h, 19); h = h * 5 + 0xe6546b64; h ^= a2; h = Utilities.RotateWithCheck(h, 19); h = h * 5 + 0xe6546b64; g ^= a1; g = Utilities.RotateWithCheck(g, 19); g = g * 5 + 0xe6546b64; g ^= a3; g = Utilities.RotateWithCheck(g, 19); g = g * 5 + 0xe6546b64; f += a4; f = Utilities.RotateWithCheck(f, 19) + 113; uint iters = ((uint)len - 1) / 20; do { uint a = Utilities.Fetch32(s); uint b = Utilities.Fetch32(s + 4); uint c = Utilities.Fetch32(s + 8); uint d = Utilities.Fetch32(s + 12); uint e = Utilities.Fetch32(s + 16); h += a; g += b; f += c; h = FarmHash.Mur(d, h) + e; g = FarmHash.Mur(c, g) + a; f = FarmHash.Mur(b + e * FarmHashConstants.c1, f) + d; f += g; g += f; s += 20; } while (--iters != 0); g = Utilities.RotateWithCheck(g, 11) * FarmHashConstants.c1; g = Utilities.RotateWithCheck(g, 17) * FarmHashConstants.c1; f = Utilities.RotateWithCheck(f, 11) * FarmHashConstants.c1; f = Utilities.RotateWithCheck(f, 17) * FarmHashConstants.c1; h = Utilities.RotateWithCheck(h + g, 19); h = h * 5 + 0xe6546b64; h = Utilities.RotateWithCheck(h, 17) * FarmHashConstants.c1; h = Utilities.RotateWithCheck(h + f, 19); h = h * 5 + 0xe6546b64; h = Utilities.RotateWithCheck(h, 17) * FarmHashConstants.c1; return(h); }
public static uint ComputeHash(byte[] s) { uint len = (uint)s.Length; if (len <= 24) { return(len <= 12 ? (len <= 4 ? Hash32Len0to4(s, len) : Hash32Len5to12(s, len)) : Hash32Len13to24(s, len)); } // len > 24 uint h = len, g = FarmHashConstants.c1 * len, f = g; uint a0 = Utilities.RotateWithCheck(Utilities.Fetch32(s, len - 4) * FarmHashConstants.c1, 17) * FarmHashConstants.c2; uint a1 = Utilities.RotateWithCheck(Utilities.Fetch32(s, len - 8) * FarmHashConstants.c1, 17) * FarmHashConstants.c2; uint a2 = Utilities.RotateWithCheck(Utilities.Fetch32(s, len - 16) * FarmHashConstants.c1, 17) * FarmHashConstants.c2; uint a3 = Utilities.RotateWithCheck(Utilities.Fetch32(s, len - 12) * FarmHashConstants.c1, 17) * FarmHashConstants.c2; uint a4 = Utilities.RotateWithCheck(Utilities.Fetch32(s, len - 20) * FarmHashConstants.c1, 17) * FarmHashConstants.c2; h ^= a0; h = Utilities.RotateWithCheck(h, 19); h = h * 5 + 0xe6546b64; h ^= a2; h = Utilities.RotateWithCheck(h, 19); h = h * 5 + 0xe6546b64; g ^= a1; g = Utilities.RotateWithCheck(g, 19); g = g * 5 + 0xe6546b64; g ^= a3; g = Utilities.RotateWithCheck(g, 19); g = g * 5 + 0xe6546b64; f += a4; f = Utilities.RotateWithCheck(f, 19) + 113; uint iters = (len - 1) / 20; int index = 0; do { uint a = Utilities.Fetch32(s, index); uint b = Utilities.Fetch32(s, index + 4); uint c = Utilities.Fetch32(s, index + 8); uint d = Utilities.Fetch32(s, index + 12); uint e = Utilities.Fetch32(s, index + 16); h += a; g += b; f += c; h = FarmHash.Mur(d, h) + e; g = FarmHash.Mur(c, g) + a; f = FarmHash.Mur(b + e * FarmHashConstants.c1, f) + d; f += g; g += f; index += 20; } while (--iters != 0); g = Utilities.RotateWithCheck(g, 11) * FarmHashConstants.c1; g = Utilities.RotateWithCheck(g, 17) * FarmHashConstants.c1; f = Utilities.RotateWithCheck(f, 11) * FarmHashConstants.c1; f = Utilities.RotateWithCheck(f, 17) * FarmHashConstants.c1; h = Utilities.RotateWithCheck(h + g, 19); h = h * 5 + 0xe6546b64; h = Utilities.RotateWithCheck(h, 17) * FarmHashConstants.c1; h = Utilities.RotateWithCheck(h + f, 19); h = h * 5 + 0xe6546b64; h = Utilities.RotateWithCheck(h, 17) * FarmHashConstants.c1; return(h); }