public void AddTwiceDoesNotChangeCount() { var hll1 = new HyperLogLog(18); hll1.Add(ulong.MaxValue); hll1.Add(ulong.MaxValue); var estimatedCount1 = hll1.Count(); var hll2 = new HyperLogLog(18); hll2.Add(ulong.MaxValue); var estimatedCount2 = hll2.Count(); Assert.Equal(estimatedCount1, estimatedCount2); }
public void HashedVsRandom(int realCardinality) { const int precision = 18; var hllRandom = new HyperLogLog(precision); for (var i = 0; i < realCardinality; i++) { var randomBytes = new byte[8]; RandomNumberGenerator.GetBytes(randomBytes); var hashValue = BitConverter.ToUInt64(randomBytes, 0); hllRandom.Add(hashValue); } var cardinalityEstimateRandom = hllRandom.Count(); var hllHashed = new HyperLogLog(precision); using (var hashAlgorithm = new SHA1CryptoServiceProvider()) { for (var i = 0; i < realCardinality; i++) { var randomBytes = new byte[12]; RandomNumberGenerator.GetBytes(randomBytes); var hashValue = HyperLogLog.Hash(hashAlgorithm, randomBytes); hllHashed.Add(hashValue); } } var cardinalityEstimateHashed = hllHashed.Count(); var percentageDifference = GetPercentageDifference(cardinalityEstimateRandom, cardinalityEstimateHashed); Assert.True(percentageDifference < ZeroFivePercent); }
public void TestHyperLogLogMerge() { var hll1 = new HyperLogLog <int>(); var hll2 = new HyperLogLog <int>(); var rand = new Random(); var tolerance = .05; HashSet <int> actual = new (); for (var i = 0; i < 5000; i++) { var k = rand.Next(20000); hll1.Add(k); actual.Add(k); } for (var i = 0; i < 5000; i++) { var k = rand.Next(20000); hll2.Add(k); actual.Add(k); } var hll = HyperLogLog <int> .Merge(hll1, hll2); hll.Cardinality().Should() .BeGreaterOrEqualTo((int)(actual.Count * (1 - tolerance))) .And .BeLessOrEqualTo((int)(actual.Count * (1 + tolerance))); }
public void Observed(TSource value) { TValue result = selector(value); hyperLogLog.Add(result); Publish(value); }
public void Add5() { var hll = new HyperLogLog(18); hll.Add(5); var estimatedCount = hll.Count(); Assert.Equal(189084, estimatedCount); }
public void AddMaxULong() { var hll = new HyperLogLog(18); hll.Add(ulong.MaxValue); var estimatedCount = hll.Count(); Assert.Equal(189084, estimatedCount); }
private void benchmarkCount(int registers) { var n = 100000; var words = Words.Dictionary(0); var m = (uint)Math.Pow(2, registers); var h = new HyperLogLog(m); foreach (var word in words) { h.Add(Encoding.ASCII.GetBytes(word)); } for (int i = 0; i < n; i++) { h.Count(); } }
static void TestHyperLogLog() { var rds = new FullRedis("127.0.0.1", null, 1); rds.Remove("ips"); var log = new HyperLogLog(rds, "ips"); XTrace.WriteLine("log.Count={0:n0}", log.Count); var count = 1_000_000; XTrace.WriteLine("准备添加[{0:n0}]个IP地址", count); Parallel.For(0, count, k => { var n = Rand.Next(); var ip = new IPAddress(n); log.Add(ip + ""); }); XTrace.WriteLine("log.Count={0:n0}", log.Count); }
public void TheRandomTheory(int realCardinality) { const int precision = 18; var hll = new HyperLogLog(precision); for (var i = 0; i < realCardinality; i++) { var randomBytes = new byte[8]; RandomNumberGenerator.GetBytes(randomBytes); var hashValue = BitConverter.ToUInt64(randomBytes, 0); hll.Add(hashValue); } var cardinalityEstimate = hll.Count(); var percentageDifference = GetPercentageDifference(realCardinality, cardinalityEstimate); Assert.True(percentageDifference < OnePercent); }
private void testHyperLogLog(int n, int lowB, int highB) { var words = Words.Dictionary(n); var bad = 0; var nWords = (UInt64)words.LongLength; var options = new ParallelOptions(); options.MaxDegreeOfParallelism = 4; Parallel.For(lowB, highB, options, i => { var m = (uint)Math.Pow(2, i); HyperLogLog h = null; try { h = new HyperLogLog(m); } catch (Exception) { Assert.Fail(string.Format("Can't make HyperLogLog({0})", m)); } foreach (var word in words) { h.Add(Encoding.ASCII.GetBytes(word)); } var expectedError = 1.04 / Math.Sqrt(m); var actualError = Math.Abs(this.geterror(nWords, h.Count())); if (actualError > expectedError) { bad++; //Assert.Fail(string.Format("Expected: {0}, Actual: {1}", expectedError, actualError)); } }); }
public void TheHashedRandomTheory(int realCardinality) { const int precision = 18; var hll = new HyperLogLog(precision); using (var hashAlgorithm = new SHA1CryptoServiceProvider()) { for (var i = 0; i < realCardinality; i++) { var randomBytes = new byte[12]; RandomNumberGenerator.GetBytes(randomBytes); var hashValue = HyperLogLog.Hash(hashAlgorithm, randomBytes); hll.Add(hashValue); } } var cardinalityEstimate = hll.Count(); var percentageDifference = GetPercentageDifference(realCardinality, cardinalityEstimate); Assert.True(percentageDifference < OnePercent); }
public void HyperLogLog_Normal() { var key = "hyper_key"; var key2 = "hyper_key2"; // 删除已有 _redis.Remove(key); var hyper = new HyperLogLog(_redis, key); _redis.SetExpire(key, TimeSpan.FromSeconds(60)); // 取出个数 var count = hyper.Count; Assert.Equal(0, count); // 添加 var vs = new[] { "1234", "abcd", "新生命团队", "ABEF" }; hyper.Add(vs); // 对比个数 var count2 = hyper.Count; Assert.Equal(count + vs.Length, count2); var hyper2 = new HyperLogLog(_redis, key2); hyper2.Add("567", "789"); var rs = hyper.Merge(key2); Assert.True(rs); Assert.Equal(vs.Length + hyper2.Count, hyper.Count); }