public void AddTwiceDoesNotChangeCount()
{
var hll1 = new HyperLogLog(18);
hll1.Add(ulong.MaxValue);
hll1.Add(ulong.MaxValue);
var estimatedCount1 = hll1.Count();
var hll2 = new HyperLogLog(18);
hll2.Add(ulong.MaxValue);
var estimatedCount2 = hll2.Count();
Assert.Equal(estimatedCount1, estimatedCount2);
}
public void HashedVsRandom(int realCardinality)
{
const int precision = 18;
var hllRandom = new HyperLogLog(precision);
for (var i = 0; i < realCardinality; i++)
{
var randomBytes = new byte[8];
RandomNumberGenerator.GetBytes(randomBytes);
var hashValue = BitConverter.ToUInt64(randomBytes, 0);
hllRandom.Add(hashValue);
}
var cardinalityEstimateRandom = hllRandom.Count();
var hllHashed = new HyperLogLog(precision);
using (var hashAlgorithm = new SHA1CryptoServiceProvider())
{
for (var i = 0; i < realCardinality; i++)
{
var randomBytes = new byte[12];
RandomNumberGenerator.GetBytes(randomBytes);
var hashValue = HyperLogLog.Hash(hashAlgorithm, randomBytes);
hllHashed.Add(hashValue);
}
}
var cardinalityEstimateHashed = hllHashed.Count();
var percentageDifference = GetPercentageDifference(cardinalityEstimateRandom, cardinalityEstimateHashed);
Assert.True(percentageDifference < ZeroFivePercent);
}
public void TestHyperLogLogMerge()
{
var hll1 = new HyperLogLog <int>();
var hll2 = new HyperLogLog <int>();
var rand = new Random();
var tolerance = .05;
HashSet <int> actual = new ();
for (var i = 0; i < 5000; i++)
{
var k = rand.Next(20000);
hll1.Add(k);
actual.Add(k);
}
for (var i = 0; i < 5000; i++)
{
var k = rand.Next(20000);
hll2.Add(k);
actual.Add(k);
}
var hll = HyperLogLog <int> .Merge(hll1, hll2);
hll.Cardinality().Should()
.BeGreaterOrEqualTo((int)(actual.Count * (1 - tolerance)))
.And
.BeLessOrEqualTo((int)(actual.Count * (1 + tolerance)));
}
public void Observed(TSource value)
{
TValue result = selector(value);
hyperLogLog.Add(result);
Publish(value);
}
public void Add5()
{
var hll = new HyperLogLog(18);
hll.Add(5);
var estimatedCount = hll.Count();
Assert.Equal(189084, estimatedCount);
}
public void AddMaxULong()
{
var hll = new HyperLogLog(18);
hll.Add(ulong.MaxValue);
var estimatedCount = hll.Count();
Assert.Equal(189084, estimatedCount);
}
private void benchmarkCount(int registers)
{
var n = 100000;
var words = Words.Dictionary(0);
var m = (uint)Math.Pow(2, registers);
var h = new HyperLogLog(m);
foreach (var word in words)
{
h.Add(Encoding.ASCII.GetBytes(word));
}
for (int i = 0; i < n; i++)
{
h.Count();
}
}
static void TestHyperLogLog()
{
var rds = new FullRedis("127.0.0.1", null, 1);
rds.Remove("ips");
var log = new HyperLogLog(rds, "ips");
XTrace.WriteLine("log.Count={0:n0}", log.Count);
var count = 1_000_000;
XTrace.WriteLine("准备添加[{0:n0}]个IP地址", count);
Parallel.For(0, count, k =>
{
var n = Rand.Next();
var ip = new IPAddress(n);
log.Add(ip + "");
});
XTrace.WriteLine("log.Count={0:n0}", log.Count);
}
public void TheRandomTheory(int realCardinality)
{
const int precision = 18;
var hll = new HyperLogLog(precision);
for (var i = 0; i < realCardinality; i++)
{
var randomBytes = new byte[8];
RandomNumberGenerator.GetBytes(randomBytes);
var hashValue = BitConverter.ToUInt64(randomBytes, 0);
hll.Add(hashValue);
}
var cardinalityEstimate = hll.Count();
var percentageDifference = GetPercentageDifference(realCardinality, cardinalityEstimate);
Assert.True(percentageDifference < OnePercent);
}
private void testHyperLogLog(int n, int lowB, int highB)
{
var words = Words.Dictionary(n);
var bad = 0;
var nWords = (UInt64)words.LongLength;
var options = new ParallelOptions();
options.MaxDegreeOfParallelism = 4;
Parallel.For(lowB, highB, options, i =>
{
var m = (uint)Math.Pow(2, i);
HyperLogLog h = null;
try
{
h = new HyperLogLog(m);
}
catch (Exception)
{
Assert.Fail(string.Format("Can't make HyperLogLog({0})", m));
}
foreach (var word in words)
{
h.Add(Encoding.ASCII.GetBytes(word));
}
var expectedError = 1.04 / Math.Sqrt(m);
var actualError = Math.Abs(this.geterror(nWords, h.Count()));
if (actualError > expectedError)
{
bad++;
//Assert.Fail(string.Format("Expected: {0}, Actual: {1}", expectedError, actualError));
}
});
}
public void TheHashedRandomTheory(int realCardinality)
{
const int precision = 18;
var hll = new HyperLogLog(precision);
using (var hashAlgorithm = new SHA1CryptoServiceProvider())
{
for (var i = 0; i < realCardinality; i++)
{
var randomBytes = new byte[12];
RandomNumberGenerator.GetBytes(randomBytes);
var hashValue = HyperLogLog.Hash(hashAlgorithm, randomBytes);
hll.Add(hashValue);
}
}
var cardinalityEstimate = hll.Count();
var percentageDifference = GetPercentageDifference(realCardinality, cardinalityEstimate);
Assert.True(percentageDifference < OnePercent);
}
public void HyperLogLog_Normal()
{
var key = "hyper_key";
var key2 = "hyper_key2";
// 删除已有
_redis.Remove(key);
var hyper = new HyperLogLog(_redis, key);
_redis.SetExpire(key, TimeSpan.FromSeconds(60));
// 取出个数
var count = hyper.Count;
Assert.Equal(0, count);
// 添加
var vs = new[] { "1234", "abcd", "新生命团队", "ABEF" };
hyper.Add(vs);
// 对比个数
var count2 = hyper.Count;
Assert.Equal(count + vs.Length, count2);
var hyper2 = new HyperLogLog(_redis, key2);
hyper2.Add("567", "789");
var rs = hyper.Merge(key2);
Assert.True(rs);
Assert.Equal(vs.Length + hyper2.Count, hyper.Count);
}