public void TestSpace() { const double eps = 0.001; const double confidence = 0.999; const int space = 2000; const int points = 100000; const bool randomized = true; var random = new Random(); var spec = new CountMinSketchSpecHashes(eps, confidence, 123456); var state = CountMinSketchStateHashes.MakeState(spec); var sent = new Dictionary <Blob, long?>(); for (var i = 0; i < points; i++) { Blob bytes; if (randomized) { bytes = TestCountMinSketchStateTopK.GenerateBytesRandom(random, space); } else { bytes = TestCountMinSketchStateTopK.GenerateBytesModulo(i, space); } state.Add(bytes.Data, 1); var count = sent.Get(bytes); if (count == null) { sent.Put(bytes, 1L); } else { sent.Put(bytes, count + 1); } if (i > 0 && i % 100000 == 0) { Console.WriteLine("Completed {0}", i); } } // compare var errors = 0; foreach (var entry in sent) { var frequency = state.EstimateCount(entry.Key.Data); if (frequency != entry.Value) { Console.WriteLine("Expected {0} received {1}", entry.Value, frequency); errors++; } } Console.WriteLine("Found {0} errors at space {1} sent {2}", errors, space, points); Assert.IsTrue(eps * points > errors); }
public static CountMinSketchState MakeState(CountMinSketchSpec spec) { CountMinSketchStateHashes hashes = CountMinSketchStateHashes.MakeState(spec.HashesSpec); CountMinSketchStateTopk topk = null; if (spec.TopkSpec != null && spec.TopkSpec > 0) { topk = new CountMinSketchStateTopk(spec.TopkSpec.Value); } return(new CountMinSketchState(hashes, topk)); }
public void TestSimpleFlow() { var state = CountMinSketchStateHashes.MakeState(GetDefaultSpec()); Add(state, "hello", 100); Assert.AreEqual(100, EstimateCount(state, "hello")); Add(state, "text", 1); Assert.AreEqual(1, EstimateCount(state, "text")); Add(state, "hello", 3); Assert.AreEqual(103, EstimateCount(state, "hello")); Assert.AreEqual(1, EstimateCount(state, "text")); }
private static void Add(CountMinSketchStateHashes state, string item, long count) { state.Add(GetBytes(item), count); }
private static long EstimateCount(CountMinSketchStateHashes state, string item) { return(state.EstimateCount(GetBytes(item))); }
public CountMinSketchState(CountMinSketchStateHashes hashes, CountMinSketchStateTopk topk) { Hashes = hashes; Topk = topk; }