public void TestHyperLogLogMerge()
{
var hll1 = new HyperLogLog <int>();
var hll2 = new HyperLogLog <int>();
var rand = new Random();
var tolerance = .05;
HashSet <int> actual = new ();
for (var i = 0; i < 5000; i++)
{
var k = rand.Next(20000);
hll1.Add(k);
actual.Add(k);
}
for (var i = 0; i < 5000; i++)
{
var k = rand.Next(20000);
hll2.Add(k);
actual.Add(k);
}
var hll = HyperLogLog <int> .Merge(hll1, hll2);
hll.Cardinality().Should()
.BeGreaterOrEqualTo((int)(actual.Count * (1 - tolerance)))
.And
.BeLessOrEqualTo((int)(actual.Count * (1 + tolerance)));
}
public void HashedVsRandom(int realCardinality)
{
const int precision = 18;
var hllRandom = new HyperLogLog(precision);
for (var i = 0; i < realCardinality; i++)
{
var randomBytes = new byte[8];
RandomNumberGenerator.GetBytes(randomBytes);
var hashValue = BitConverter.ToUInt64(randomBytes, 0);
hllRandom.Add(hashValue);
}
var cardinalityEstimateRandom = hllRandom.Count();
var hllHashed = new HyperLogLog(precision);
using (var hashAlgorithm = new SHA1CryptoServiceProvider())
{
for (var i = 0; i < realCardinality; i++)
{
var randomBytes = new byte[12];
RandomNumberGenerator.GetBytes(randomBytes);
var hashValue = HyperLogLog.Hash(hashAlgorithm, randomBytes);
hllHashed.Add(hashValue);
}
}
var cardinalityEstimateHashed = hllHashed.Count();
var percentageDifference = GetPercentageDifference(cardinalityEstimateRandom, cardinalityEstimateHashed);
Assert.True(percentageDifference < ZeroFivePercent);
}
public void CountWithoutAdd()
{
var hll = new HyperLogLog(18);
var estimatedCount = hll.Count();
Assert.Equal(189083, estimatedCount);
}
public void SerializeTo(HyperLogLog hyperLogLog, Stream stream)
{
using (var bw = new BinaryWriter(stream))
{
bw.Write((byte)hyperLogLog.B);
bw.Write(hyperLogLog._registers);
bw.Flush();
}
}
public void AddMaxULong()
{
var hll = new HyperLogLog(18);
hll.Add(ulong.MaxValue);
var estimatedCount = hll.Count();
Assert.Equal(189084, estimatedCount);
}
public void SerializeTo(HyperLogLog hyperLogLog, Stream stream)
{
using (var bw = new BinaryWriter(stream))
{
bw.Write((byte)hyperLogLog.B);
bw.Write(hyperLogLog._registers);
bw.Flush();
}
}
public void Add5()
{
var hll = new HyperLogLog(18);
hll.Add(5);
var estimatedCount = hll.Count();
Assert.Equal(189084, estimatedCount);
}
private static HyperLogLog CreateHyperLogLogWithHashedDateTimes(int n, byte b, int start = 0)
{
var hyperLogLog = new HyperLogLog(b);
for (Int64 i = start; i < start + n; i++)
{
hyperLogLog.AddDateTime(new DateTime(2000, 1, 1).AddMinutes(i));
}
return(hyperLogLog);
}
private static HyperLogLog CreateHyperLogLogWithHashedGuids(int n, byte b, int start = 0)
{
var hyperLogLog = new HyperLogLog(b);
for (Int64 i = start; i < start + n; i++)
{
hyperLogLog.AddGuid(Guid.NewGuid());
}
return(hyperLogLog);
}
private static HyperLogLog CreateHyperLogLogWithHashedIntegers64(int n, byte b, int start = 0)
{
var hyperLogLog = new HyperLogLog(b);
for (Int64 i = start; i < start + n; i++)
{
hyperLogLog.AddInt64(i);
}
return(hyperLogLog);
}
private static HyperLogLog CreateHyperLogLogWithHashedIntegers16(short n, byte b)
{
var hyperLogLog = new HyperLogLog(b);
for (Int16 i = 0; i < n; i++)
{
hyperLogLog.AddInt64(i);
}
return(hyperLogLog);
}
public void HashConstantWithMD5()
{
const ulong value = 0x2D51AF5C52FDE6B4ul;
using (var hashAlgorithm = new MD5CryptoServiceProvider())
{
var hashValue = HyperLogLog.Hash(hashAlgorithm, BitConverter.GetBytes(value));
Assert.Equal(16663394367412432550ul, hashValue);
}
}
private static void Test_CalculateEstimatedCount_ShouldBeWithinAcceptableErrorRange(
HyperLogLog hyperLogLog,
int n,
double acceptablePercentError)
{
// Act
int estimatedCount = hyperLogLog.CalculateEstimatedCount();
// Assert
Assert.That(estimatedCount, Is.EqualTo(n).Within(acceptablePercentError).Percent);
}
private static HyperLogLog CreateHyperLogLogWithHashedStrings(int n, byte b)
{
HyperLogLog hyperLogLog = new HyperLogLog(b);
for (int i = 0; i < n; i++)
{
hyperLogLog.AddUTF8String(i.ToString(CultureInfo.InvariantCulture));
}
return(hyperLogLog);
}
public void HashConstantWithSHA1()
{
const ulong value = 0x2D51AF5C52FDE6B4ul;
using (var hashAlgorithm = new SHA1CryptoServiceProvider())
{
var hashValue = HyperLogLog.Hash(hashAlgorithm, BitConverter.GetBytes(value));
Assert.Equal(17851087020509344997ul, hashValue);
}
}
public void Merge_NullSet_ThrowsExeption()
{
// Arrange
var hyperLogLog = new HyperLogLog(4);
// Act
TestDelegate action = () => hyperLogLog.Merge(null);
// Assert
ArgumentNullException exception = Assert.Throws <ArgumentNullException>(action);
Assert.AreEqual("other", exception.ParamName);
}
public void Merge_SameSet_ThrowsExeption()
{
// Arrange
var hyperLogLog = new HyperLogLog(4);
// Act
TestDelegate action = () => hyperLogLog.Merge(hyperLogLog);
// Assert
ArgumentException exception = Assert.Throws <ArgumentException>(action);
Assert.AreEqual("other", exception.ParamName);
StringAssert.StartsWith("Cannot merge instance of HyperLogLog to itself", exception.Message);
}
public void Merge_DifferentB_ThrowsExeption()
{
// Arrange
var hyperLogLog = new HyperLogLog(4);
var other = new HyperLogLog(5);
// Act
TestDelegate action = () => hyperLogLog.Merge(other);
// Assert
ArgumentException exception = Assert.Throws <ArgumentException>(action);
Assert.AreEqual("other", exception.ParamName);
StringAssert.StartsWith("Cannot merge instance of HyperLogLog with b = 5 to instance with b = 4", exception.Message);
}
public HyperLogLog DeserializeTo(Stream stream, HashAlgorithm hashAlgorithm, IBytesConverter bytesConverter)
{
using (var br = new BinaryReader(stream))
{
var b = br.ReadByte();
var result = new HyperLogLog(hashAlgorithm, bytesConverter, b);
for (int i = 0; i < result.M; i++)
{
result._registers[i] = br.ReadByte();
}
return(result);
}
}
public void AddTwiceDoesNotChangeCount()
{
var hll1 = new HyperLogLog(18);
hll1.Add(ulong.MaxValue);
hll1.Add(ulong.MaxValue);
var estimatedCount1 = hll1.Count();
var hll2 = new HyperLogLog(18);
hll2.Add(ulong.MaxValue);
var estimatedCount2 = hll2.Count();
Assert.Equal(estimatedCount1, estimatedCount2);
}
public HyperLogLog DeserializeTo(Stream stream, HashAlgorithm hashAlgorithm, IBytesConverter bytesConverter)
{
using (var br = new BinaryReader(stream))
{
var b = br.ReadByte();
var result = new HyperLogLog(hashAlgorithm, bytesConverter, b);
for (int i =0; i < result.M; i++)
{
result._registers[i] = br.ReadByte();
}
return result;
}
}
public void SerializeDeserialize_SameEstimate(int n, byte b)
{
// Arrange
HyperLogLog hyperLogLog = CreateHyperLogLogWithHashedStrings(n, b);
BinaryFormatter formatter = new BinaryFormatter();
MemoryStream stream = new MemoryStream();
// Act
formatter.Serialize(stream, hyperLogLog);
stream.Position = 0;
HyperLogLog deserialized = (HyperLogLog)formatter.Deserialize(stream);
// Assert
int originalEstimate = hyperLogLog.CalculateEstimatedCount();
Assert.That(originalEstimate, Is.GreaterThan(0.9 * n));
Assert.That(originalEstimate, Is.EqualTo(deserialized.CalculateEstimatedCount()));
}
private void benchmarkCount(int registers)
{
var n = 100000;
var words = Words.Dictionary(0);
var m = (uint)Math.Pow(2, registers);
var h = new HyperLogLog(m);
foreach (var word in words)
{
h.Add(Encoding.ASCII.GetBytes(word));
}
for (int i = 0; i < n; i++)
{
h.Count();
}
}
public void TheRandomTheory(int realCardinality)
{
const int precision = 18;
var hll = new HyperLogLog(precision);
for (var i = 0; i < realCardinality; i++)
{
var randomBytes = new byte[8];
RandomNumberGenerator.GetBytes(randomBytes);
var hashValue = BitConverter.ToUInt64(randomBytes, 0);
hll.Add(hashValue);
}
var cardinalityEstimate = hll.Count();
var percentageDifference = GetPercentageDifference(realCardinality, cardinalityEstimate);
Assert.True(percentageDifference < OnePercent);
}
static void TestHyperLogLog()
{
var rds = new FullRedis("127.0.0.1", null, 1);
rds.Remove("ips");
var log = new HyperLogLog(rds, "ips");
XTrace.WriteLine("log.Count={0:n0}", log.Count);
var count = 1_000_000;
XTrace.WriteLine("准备添加[{0:n0}]个IP地址", count);
Parallel.For(0, count, k =>
{
var n = Rand.Next();
var ip = new IPAddress(n);
log.Add(ip + "");
});
XTrace.WriteLine("log.Count={0:n0}", log.Count);
}
private void testHyperLogLog(int n, int lowB, int highB)
{
var words = Words.Dictionary(n);
var bad = 0;
var nWords = (UInt64)words.LongLength;
var options = new ParallelOptions();
options.MaxDegreeOfParallelism = 4;
Parallel.For(lowB, highB, options, i =>
{
var m = (uint)Math.Pow(2, i);
HyperLogLog h = null;
try
{
h = new HyperLogLog(m);
}
catch (Exception)
{
Assert.Fail(string.Format("Can't make HyperLogLog({0})", m));
}
foreach (var word in words)
{
h.Add(Encoding.ASCII.GetBytes(word));
}
var expectedError = 1.04 / Math.Sqrt(m);
var actualError = Math.Abs(this.geterror(nWords, h.Count()));
if (actualError > expectedError)
{
bad++;
//Assert.Fail(string.Format("Expected: {0}, Actual: {1}", expectedError, actualError));
}
});
}
public void TheHashedRandomTheory(int realCardinality)
{
const int precision = 18;
var hll = new HyperLogLog(precision);
using (var hashAlgorithm = new SHA1CryptoServiceProvider())
{
for (var i = 0; i < realCardinality; i++)
{
var randomBytes = new byte[12];
RandomNumberGenerator.GetBytes(randomBytes);
var hashValue = HyperLogLog.Hash(hashAlgorithm, randomBytes);
hll.Add(hashValue);
}
}
var cardinalityEstimate = hll.Count();
var percentageDifference = GetPercentageDifference(realCardinality, cardinalityEstimate);
Assert.True(percentageDifference < OnePercent);
}
public void HyperLogLog_Normal()
{
var key = "hyper_key";
var key2 = "hyper_key2";
// 删除已有
_redis.Remove(key);
var hyper = new HyperLogLog(_redis, key);
_redis.SetExpire(key, TimeSpan.FromSeconds(60));
// 取出个数
var count = hyper.Count;
Assert.Equal(0, count);
// 添加
var vs = new[] { "1234", "abcd", "新生命团队", "ABEF" };
hyper.Add(vs);
// 对比个数
var count2 = hyper.Count;
Assert.Equal(count + vs.Length, count2);
var hyper2 = new HyperLogLog(_redis, key2);
hyper2.Add("567", "789");
var rs = hyper.Merge(key2);
Assert.True(rs);
Assert.Equal(vs.Length + hyper2.Count, hyper.Count);
}
public void TestNewDefaultHyperLogLog()
{
var hll = HyperLogLog.NewDefaultHyperLogLog(0.1);
Assert.AreEqual(128u, hll.M);
}
public void ConstructHyperLogLog4DoesNotThrow()
{
var hll = new HyperLogLog(7);
}
public CardinalityEstimateImpl(Func <TSource, TValue> selector, double stdError)
{
this.selector = selector;
hyperLogLog = new HyperLogLog(stdError);
}
public void ConstructHyperLogLog18DoesNotThrow()
{
var hll = new HyperLogLog(18);
}
