public void testHashCodesM3_128_ints()
{
int seed = 123;
Random rand = new Random(seed);
HashFunction hf = Hashing.murmur3_128(seed);
for (int i = 0; i < 1000; i++)
{
int val = rand.nextInt();
byte[] data = ByteBuffer.allocate(4).putInt(val).array();
// guava stores the hashcodes in little endian order
ByteBuffer buf = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN);
buf.put(hf.hashBytes(data).asBytes());
buf.flip();
long gl1 = buf.getLong();
long gl2 = buf.getLong(8);
long[] hc = Murmur3.hash128(data, 0, data.length, seed);
long m1 = hc[0];
long m2 = hc[1];
Assert.Equal(gl1, m1);
Assert.Equal(gl2, m2);
byte[] offsetData = new byte[data.length + 50];
Array.Copy(data, 0, offsetData, 50, data.length);
hc = Murmur3.hash128(offsetData, 50, data.length, seed);
Assert.Equal(gl1, hc[0]);
Assert.Equal(gl2, hc[1]);
}
}
/// <summary>
/// Computes the custom checksum of the data.
/// </summary>
/// <param name="data">the data to compute the checksum for.</param>
/// <param name="checksum1">the 64 bit component of this checksum</param>
/// <param name="checksum2">the 32 bit component of this checksum</param>
/// <param name="length">the number of bytes to have in the checksum,Must </param>
/// <remarks>This checksum is similiar to Adler</remarks>
public static void ComputeChecksum(IntPtr data, out long checksum1, out int checksum2, int length)
{
Stats.ChecksumCount++;
Murmur3.ComputeHash((byte *)data, length, out ulong a, out ulong b);
checksum1 = (long)a;
checksum2 = (int)b ^ (int)(b >> 32);
}
private void Reset()
{
_hash1 = 0;
_hash2 = 0;
_totalBytesProcessed = 0;
Murmur3.Return(this);
}
public override Routee Select(object message, Routee[] routees)
{
if (message == null)
{
return(NoRoutee.NoRoutee);
}
if (_hashMapping.ContainsKey(message.GetType()))
{
var key = _hashMapping[message.GetType()](message);
if (key == null)
{
return(NoRoutee.NoRoutee);
}
var hash = Murmur3.Hash(key);
return(routees[Math.Abs(hash) % routees.Length]); //The hash might be negative, so we have to make sure it's positive
}
else if (message is ConsistentHashable)
{
var hashable = (ConsistentHashable)message;
int hash = Murmur3.Hash(hashable.ConsistentHashKey);
return(routees[Math.Abs(hash) % routees.Length]); //The hash might be negative, so we have to make sure it's positive
}
else
{
_log.Value.Warn("Message [{0}] must be handled by hashMapping, or implement [{1}] or be wrapped in [{2}]", message.GetType().Name, typeof(ConsistentHashable).Name, typeof(ConsistentHashableEnvelope).Name);
return(NoRoutee.NoRoutee);
}
}
/// <summary>
/// Calculate the the hash-code for a horizontal line
/// </summary>
/// <param name="y">int with y coordinate</param>
/// <param name="right">optional x starting coordinate of the hash calculation</param>
/// <param name="left">optional x ending coordinate of the hash calculation</param>
/// <returns>uint with the hash</returns>
public uint HorizontalHash(int y, int?right = null, int?left = null)
{
var offset = (left ?? Left) * BytesPerPixel + y * Stride;
var length = (right ?? Right) - (left ?? Left) * BytesPerPixel;
var hash = new Murmur3(Seed, (uint)length);
while (length >= 4)
{
hash.AddBytes(Pointer[offset++], Pointer[offset++], Pointer[offset++], Pointer[offset++]);
length -= 4;
}
switch (length)
{
case 3:
hash.AddTrailingBytes(Pointer[offset++], Pointer[offset++], Pointer[offset]);
break;
case 2:
hash.AddTrailingBytes(Pointer[offset++], Pointer[offset]);
break;
case 1:
hash.AddTrailingBytes(Pointer[offset]);
break;
}
return(hash.CalculatedHash);
}
public unsafe void TestIsSame()
{
byte[] data = new byte[4096];
Random r = new Random();
for (int x = 0; x < 100; x++)
{
r.NextBytes(data);
Murmur3Orig mm3 = new Murmur3Orig();
byte[] checksum = mm3.ComputeHash(data);
byte[] checksum2 = new byte[16];
fixed(byte *lp = data)
{
ulong value1;
ulong value2;
Murmur3.ComputeHash(lp, data.Length, out value1, out value2);
Array.Copy(BitConverter.GetBytes(value1), 0, checksum2, 0, 8);
Array.Copy(BitConverter.GetBytes(value2), 0, checksum2, 8, 8);
}
Assert.IsTrue(checksum2.SequenceEqual(checksum));
}
}
private string CalculateHash(int[] raw_picture)
{
byte[] raw_picture_byte = new byte[raw_picture.Length * sizeof(int)];
Buffer.BlockCopy(raw_picture, 0, raw_picture_byte, 0, raw_picture_byte.Length);
byte[] hash = new Murmur3().ComputeHash(raw_picture_byte);
return(Convert.ToBase64String(hash));
}
public void Should_outperform_MD5_average_case_performance()
{
//going to use a reasonably large string, because it forces additional cleanup work to occur in the tail
var testBytes = Encoding.Unicode.GetBytes("*****@*****.**");
var targetItems = 1000000; //1 million hash attempts
var i = targetItems;
var start = DateTime.UtcNow.Ticks;
while (i > 0)
{
Murmur3.Hash(testBytes);
i--;
}
var end = DateTime.UtcNow.Ticks;
var murmur3Elapsed = new TimeSpan(end - start);
start = DateTime.UtcNow.Ticks;
var m5 = MD5.Create();
i = targetItems;
while (i > 0)
{
m5.ComputeHash(testBytes);
i--;
}
end = DateTime.UtcNow.Ticks;
var md5Elapsed = new TimeSpan(end - start);
Assert.IsTrue(murmur3Elapsed.Ticks < md5Elapsed.Ticks, "Expected Murmur3 to be faster than MD5");
Assert.Pass("Murmur3 completed in {0} seconds; MD5 in {1} seconds", murmur3Elapsed.TotalSeconds, md5Elapsed.TotalSeconds);
}
public void ComputeHash32Test(byte[] data, uint seed, uint expected)
{
Murmur3 hasher = new Murmur3();
uint actual = hasher.ComputeHash32(data, seed);
Assert.Equal(expected, actual);
}
public void When_performing_the_standard_test()
{
// Adapted from: https://code.google.com/p/smhasher/source/browse/trunk/KeysetTest.cpp#13
const uint Murmur3_x64_128 = 0x6384BA69u;
using (var hash = new Murmur3())
// Also test that Merkle incremental hashing works.
using (var cs = new CryptoStream(Stream.Null, hash, CryptoStreamMode.Write))
{
var key = new byte[256];
for (var i = 0; i < 256; i++)
{
key[i] = (byte)i;
using (var m = new Murmur3(256 - i))
{
var computed = m.ComputeHash(key, 0, i);
// Also check that your implementation deals with incomplete
// blocks.
cs.Write(computed, 0, 5);
cs.Write(computed, 5, computed.Length - 5);
}
}
cs.FlushFinalBlock();
var final = hash.Hash;
var verification = ((uint)final[0]) | ((uint)final[1] << 8) | ((uint)final[2] << 16) | ((uint)final[3] << 24);
Assert.Equal(Murmur3_x64_128, verification);
}
}
public void Murmur3Characteristics()
{
var hashF = Murmur3.Create();
var hash = hashF.ComputeHash(Encoding.ASCII.GetBytes("9089089whatevathisstringshouldbe123!!!"));
Assert.Equal(8, hash.Length);
}
public static uint Murmur32(this IEnumerable <byte> value, uint seed)
{
using (Murmur3 murmur = new Murmur3(seed))
{
return(murmur.ComputeHash(value.ToArray()).ToUInt32(0));
}
}
public void Murmur3SimilarStrings()
{
var hash = Murmur3.Create();
var hash1 = hash.ComputeHash(Encoding.ASCII.GetBytes("awesome1"));
var hash2 = hash.ComputeHash(Encoding.ASCII.GetBytes("awesome1."));
Assert.NotEqual(hash1, hash2);
}
public void GenerateHash128(string text, string expectedHash)
{
var bytes = Encoding.ASCII.GetBytes(text);
var hash = HashGenerator.GenerateHash128(bytes, Murmur3.Get128BitHashFunction());
hash.ToString().Should().Be(expectedHash);
}
public void GenerateHash32(string text, int expectedHash)
{
var bytes = Encoding.ASCII.GetBytes(text);
var hash = HashGenerator.GenerateHash32(bytes, Murmur3.Get32BitHashFunction());
hash.AsInt32().Should().Be(expectedHash);
}
private uint TestHash(string testString, uint seed)
{
var hashAlgoritm = new Murmur3(seed);
var testBytes = Encoding.UTF8.GetBytes(testString);
var hash = hashAlgoritm.ComputeHash(testBytes);
return(hash.ToUInt32());
}
private static Func <string, Node> CreateNodeFunc(string value)
{
var nodeType = EvaluateNodeType(value);
var namespaceString = CreateNamespaceString(value);
var id = Murmur3.Hash(value).ToString();
return(key => new Node(id, nodeType, namespaceString));
}
public unsafe void Benchmark()
{
byte[] data = new byte[4096];
Random r = new Random(1);
r.NextBytes(data);
//Prime the run
Murmur3Orig mm3 = new Murmur3Orig();
for (int x = 0; x < 1000; x++)
{
mm3.ComputeHash(data);
fixed(byte *lp = data)
{
for (int x = 0; x < 1000; x++)
{
Murmur3.ComputeHash(lp, data.Length, out ulong value1, out ulong value2);
}
for (int x = 0; x < 1000; x++)
{
Footer.ComputeChecksum((IntPtr)lp, out long value3, out int value4, data.Length);
}
}
Stopwatch sw1 = new Stopwatch();
Stopwatch sw2 = new Stopwatch();
Stopwatch sw3 = new Stopwatch();
sw1.Start();
for (int x = 0; x < 10000; x++)
mm3.ComputeHash(data); }
sw1.Stop();
fixed(byte *lp = data)
{
sw2.Start();
for (int x = 0; x < 10000; x++)
{
Murmur3.ComputeHash(lp, data.Length, out ulong value1, out ulong value2);
}
sw2.Stop();
sw3.Start();
for (int x = 0; x < 10000; x++)
{
Footer.ComputeChecksum((IntPtr)lp, out long value3, out int value4, data.Length);
}
sw3.Stop();
}
System.Console.WriteLine("orig: " + (4096 * 10000 / sw1.Elapsed.TotalSeconds / 1024 / 1024).ToString("0 MB/S"));
System.Console.WriteLine("mine: " + (4096 * 10000 / sw2.Elapsed.TotalSeconds / 1024 / 1024).ToString("0 MB/S"));
System.Console.WriteLine("old: " + (4096 * 10000 / sw3.Elapsed.TotalSeconds / 1024 / 1024).ToString("0 MB/S"));
}
public void Should_compute_equivalent_hash_value_for_byte_array()
{
var str1 = Encoding.UTF8.GetBytes("this is a string");
var str2 = Encoding.UTF8.GetBytes("this is a string");
var hash1 = Murmur3.HashBytes(str1);
var hash2 = Murmur3.HashBytes(str2);
Assert.AreEqual(hash1, hash2);
}
public void Should_compute_equivalent_hash_value_for_strings()
{
var str1 = "this is a string";
var str2 = "this is a string";
var hash1 = Murmur3.Hash(str1);
var hash2 = Murmur3.Hash(str2);
Assert.AreEqual(hash1, hash2);
}
public void Should_compute_equivalent_hash_value_for_primitives()
{
var int1 = 1;
var int2 = 1;
var hash1 = Murmur3.Hash(int1);
var hash2 = Murmur3.Hash(int2);
Assert.AreEqual(hash1, hash2);
}
public void Should_NOT_compute_equivalent_hash_value_for_byte_equivalent_primitives()
{
var int1 = 1;
var bool1 = true;
//bitwise, these two values should be equivalent
//HOWEVER, Murmur3 takes the byte-array length into account as part of its hash.
//thus, the values SHOULD NOT BE EQUIVALENT
var hash1 = Murmur3.Hash(int1);
var hash2 = Murmur3.Hash(bool1);
Assert.AreNotEqual(hash1, hash2);
}
protected override uint GetAppearanceHash(uint Seed)
{
// calculate hash
Murmur3 hash = new Murmur3(Seed);
hash.Step((uint)(RenderInfo.Dimension.X));
hash.Step((uint)(RenderInfo.Dimension.Y));
hash.Step((uint)(HotspotIndex));
// additional to generic values: add flags
hash.Step(dataSource.Flags.Value);
return(hash.Finish());
}
public void testHashCodesM3_32_double()
{
int seed = 123;
Random rand = new Random(seed);
HashFunction hf = Hashing.murmur3_32(seed);
for (int i = 0; i < 1000; i++)
{
double val = rand.nextDouble();
byte[] data = ByteBuffer.allocate(8).putDouble(val).array();
int hc1 = hf.hashBytes(data).asInt();
int hc2 = Murmur3.hash32(data, data.length, seed);
Assert.Equal(hc1, hc2);
}
}
public void testHashCodesM3_32_string()
{
string key = "test";
int seed = 123;
HashFunction hf = Hashing.murmur3_32(seed);
int hc1 = hf.hashBytes(key.getBytes()).asInt();
int hc2 = Murmur3.hash32(key.getBytes(), key.getBytes().length, seed);
Assert.Equal(hc1, hc2);
key = "testkey";
hc1 = hf.hashBytes(key.getBytes()).asInt();
hc2 = Murmur3.hash32(key.getBytes(), key.getBytes().length, seed);
Assert.Equal(hc1, hc2);
}
public void Should_compute_equivalent_hash_values_for_equivalent_POCOs()
{
var poco1 = new TestPoco()
{
Name = "Aaron", Value = 1337
};
var poco2 = new TestPoco()
{
Name = "Aaron", Value = 1337
};
var hash1 = Murmur3.Hash(poco1);
var hash2 = Murmur3.Hash(poco2);
Assert.AreEqual(hash1, hash2);
}
public ulong CalculateHash()
{
const float position_granularity = 0.001f; // millimeter
const float normal_granularity = 0.00001f;
const float color_granularity = 0.001f;
const float uv_granularity = 0.0001f;
var hasher = new Murmur3();
hasher.AddHash(m_posSet);
hasher.AddHash(m_normalSet);
hasher.AddHash(m_uv1Set);
hasher.AddHash(m_uv2Set);
hasher.AddHash(m_uv3Set);
if (m_posSet)
{
hasher.AddHash(m_pos, position_granularity);
}
if (m_normalSet)
{
hasher.AddHash(m_normal, normal_granularity);
}
hasher.AddHash(m_color, color_granularity);
if (m_uv1Set)
{
hasher.AddHash(m_uv1, uv_granularity);
}
if (m_uv2Set)
{
hasher.AddHash(m_uv2, uv_granularity);
}
if (m_uv3Set)
{
hasher.AddHash(m_uv3, uv_granularity);
}
hasher.Finish();
return(hasher.Hash64Bit);
}
/// <summary>
/// <see cref="IHashProvider"/> factory.
/// </summary>
/// <param name="func">Hash function to use; see <see cref="HashFunc"/>.</param>
/// <returns></returns>
private IHashProvider CreateHashAlgorithm(HashFunc func)
{
switch (func)
{
case HashFunc.SHA256:
return(SHA256Provider.Create());
case HashFunc.SHA512:
return(SHA512Provider.Create());
case HashFunc.Murmur3:
return(Murmur3.Create());
default:
throw new NotImplementedException();
}
}
public void Should_have_low_number_of_Guid_collisions()
{
var targetItems = 1000000; //10 million guids
var i = targetItems;
var start = DateTime.UtcNow.Ticks;
var hashes = new HashSet <int>();
while (i > 0)
{
var hash = Murmur3.Hash(Guid.NewGuid());
hashes.Add(hash);
i--;
}
var end = DateTime.UtcNow.Ticks;
var timespan = new TimeSpan(end - start);
var epsilon = (double)(targetItems - hashes.Count) / targetItems;
Assert.IsTrue(epsilon <= 0.00125d, string.Format("Larger number of hash collisions ({0}%) than acceptable.", epsilon * 100));
Assert.Pass("Test completed in {0} seconds with {1}% collisions", timespan.TotalSeconds, epsilon * 100);
}
UInt64 maxCacheSizeBytes; //Max. size of the cache in bytes
public SHARDSGhostCache(UInt32 cacheBlockSize, UInt64 maxCacheSizeBytes, bool noisyOutput)
{
Debug.Assert(maxCacheSizeBytes % cacheBlockSize == 0);
UInt64 num_blocks_req = maxCacheSizeBytes / cacheBlockSize;
//if (num_blocks_req > Int32.MaxValue)
//{
// Console.WriteLine("StackDepthGhostCache FAILED to initialize; {0} blocks of size {1} == {2} bytes total is too large",
// num_blocks_req, cacheBlockSize, maxCacheSizeBytes);
// Debug.Assert(0 == 1);
//}
//this.totalAvailCacheBlocks = (Int32)num_blocks_req;
this.totalAvailCacheBlocks = (Int64)num_blocks_req;
//set the sampling parameters
this.modulusP = 16777216; //2^24, as specified in the paper
this.modulusPPower = 24; // power of the modulus (for efficient reuse distance scaling)
this.thresholdT = 167772; // yields sampling rate of 0.01
this.samplingRateR = (double)this.thresholdT / (double)this.modulusP; //just in case we need the sampling rate in a local variable
this.Murmur3HashFunction = new Murmur3();
this.distanceTree = new SortedList <UInt64, GhostCacheElement>(new ReverseComparer());
this.hashTableLookup = new Dictionary <UInt64, GhostCacheElement>();
this.refreshCounter = 0;
this.cacheBlockSize = cacheBlockSize;
this.maxCacheSizeBytes = maxCacheSizeBytes;
this.totalCacheReferences = 0;
this.totalProcessedReferences = 0;
this.reuseAccessHistogram = new UInt32[(Int64)num_blocks_req];
//Initialize counters to 0
for (Int64 i = 0; i < this.totalAvailCacheBlocks; i++)
{
this.reuseAccessHistogram[i] = 0;
}
//Array.Clear(this.reuseAccessHistogram, 0, (Int32)num_blocks_req);
this.noisyOutput = noisyOutput;
}
public void testHashCodeM3_64()
{
byte[] origin = Encoding.UTF8.GetBytes(
"It was the best of times, it was the worst of times," +
" it was the age of wisdom, it was the age of foolishness," +
" it was the epoch of belief, it was the epoch of incredulity," +
" it was the season of Light, it was the season of Darkness," +
" it was the spring of hope, it was the winter of despair," +
" we had everything before us, we had nothing before us," +
" we were all going direct to Heaven," +
" we were all going direct the other way.");
long hash = Murmur3.hash64(origin, 0, origin.Length);
Assert.Equal(305830725663368540L, hash);
byte[] originOffset = new byte[origin.Length + 150];
Arrays.fill(originOffset, (byte)123);
Array.Copy(origin, 0, originOffset, 150, origin.Length);
hash = Murmur3.hash64(originOffset, 150, origin.Length);
Assert.Equal(305830725663368540L, hash);
}
public static string Hash(string input)
{
if (string.IsNullOrEmpty(input))
{
return null;
}
var algorithm = new Murmur3();
var rawBytes = Encoding.UTF8.GetBytes(input);
var hashedBytes = algorithm.ComputeHash(rawBytes);
return Convert.ToBase64String(hashedBytes);
var sb = new StringBuilder(64);
foreach (var x in hashedBytes)
{
sb.AppendFormat("{0:x2}", x);
}
return sb.ToString();
}
/// <summary>
/// Perform minification of the javascript.
/// It must be a valid javascript code, without <script> tags
/// </summary>
private static string MinifyJavascript(string javascript)
{
// return javascript;
if (javascript == null || javascript == "") {
return javascript;
}
Murmur3 mrmr = new Murmur3();
string hash = mrmr.ComputeHashBase64(Encoding.UTF8.GetBytes(javascript));
object cachedScript = minifiedCache.Get(hash);
cachedScript = null;
if (cachedScript != null) {
// the script has already been minified, return cached version
return cachedScript.ToString();
} else {
// there is no cached version, minify...
Minifier minifier = new Minifier();
// use default asp.net minify settings
CodeSettings codeSettings = new CodeSettings() { EvalTreatment = EvalTreatment.MakeImmediateSafe, PreserveImportantComments = false };
string minifiedScript = minifier.MinifyJavaScript(javascript, codeSettings);
if (minifiedScript == null || minifier.ErrorList.Count > 0) {
// there has been an error(s) in the minify process (probably a js syntax error)
// resetting to the original script
minifiedScript = javascript;
}
// insert minified version into the cache
// even if there has been an error, we still are benefiting from the cache (there is no point trying to minify every time just to get an error repeatedly)
minifiedCache.Set(hash, minifiedScript, DateTime.Now.AddMinutes(60));
return minifiedScript;
}
}
/// <summary>
/// Asynchronously attempts to read a stream and return the symbol metadata from it.
/// </summary>
/// <param name="stream">The stream to read the metadata from.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>
/// A <see cref="Task{ISymbolMetadata}" /> that represents the pending read operation. Once the task completes
/// <see cref="Task{ISymbolMetadata}.Result" /> will contain the metadata if it was successfully read, or <c>null</c> if the
/// metadata could not be read.
/// </returns>
public async Task<ISymbolMetadata> TryGetSymbolMetadataAsync(
Stream stream, CancellationToken cancellationToken = default(CancellationToken))
{
using (var hash = new Murmur3())
using (var cs = new CryptoStream(Stream.Null, hash, CryptoStreamMode.Write))
{
await stream.CopyToAsync(cs, Environment.SystemPageSize, cancellationToken);
cs.FlushFinalBlock();
var final = hash.Hash;
var s = new StringBuilder(32);
for (var i = 0; i < 16; i++)
s.Append(final[i].ToString("x2", CultureInfo.InvariantCulture));
return new FallbackSymbolMetadata(s.ToString());
}
}
