public void DeserializedEstimatorUsesSameHashAsOriginal()
{
// Prepare some elements
IList <int> elements = new List <int>();
for (int i = 0; i < 150; i++)
{
elements.Add(Rand.Next());
}
foreach (HashFunctionId hashFunctionId in Enum.GetValues(typeof(HashFunctionId)))
{
// Add elements to an estimator using the given hashFunctionId
CardinalityEstimator original = new CardinalityEstimator(hashFunctionId: hashFunctionId);
foreach (int element in elements)
{
original.Add(element);
}
// Serialize
var serializer = new CardinalityEstimatorSerializer();
byte[] results;
using (var memoryStream = new MemoryStream())
{
serializer.Serialize(memoryStream, original, false);
results = memoryStream.ToArray();
}
// Deserialize
CardinalityEstimator deserialized;
using (var memoryStream = new MemoryStream(results))
{
deserialized = serializer.Deserialize(memoryStream, false);
}
// Add the elements again, should have no effect on state
foreach (int element in elements)
{
deserialized.Add(element);
}
Assert.AreEqual(original.Count(), deserialized.Count());
}
}
private void TestDeserializer(int cardinality)
{
CardinalityEstimator hll = CreateAndFillCardinalityEstimator(cardinality);
CardinalityEstimator hll2;
var serializer = new CardinalityEstimatorSerializer();
byte[] results;
using (var memoryStream = new MemoryStream())
{
serializer.Serialize(memoryStream, hll);
results = memoryStream.ToArray();
}
using (var memoryStream = new MemoryStream(results))
{
hll2 = serializer.Deserialize(memoryStream);
}
CardinalityEstimatorState data = hll.GetState();
CardinalityEstimatorState data2 = hll2.GetState();
Assert.AreEqual(data.BitsPerIndex, data2.BitsPerIndex);
Assert.AreEqual(data.IsSparse, data2.IsSparse);
Assert.IsTrue((data.DirectCount != null && data2.DirectCount != null) || (data.DirectCount == null && data2.DirectCount == null));
Assert.IsTrue((data.LookupSparse != null && data2.LookupSparse != null) ||
(data.LookupSparse == null && data2.LookupSparse == null));
Assert.IsTrue((data.LookupDense != null && data2.LookupDense != null) || (data.LookupDense == null && data2.LookupDense == null));
if (data.DirectCount != null)
{
// DirectCount are subsets of each-other => they are the same set
Assert.IsTrue(data.DirectCount.IsSubsetOf(data2.DirectCount) && data2.DirectCount.IsSubsetOf(data.DirectCount));
}
if (data.LookupSparse != null)
{
Assert.IsTrue(data.LookupSparse.DictionaryEqual(data2.LookupSparse));
}
if (data.LookupDense != null)
{
Assert.IsTrue(data.LookupDense.SequenceEqual(data2.LookupDense));
}
}
public void TestSerializerMultipleCardinalityAndBitsCombinations()
{
for (int bits = 4; bits <= 16; bits++)
{
for (int cardinality = 1; cardinality <= 1000; cardinality++)
{
var estimator = CreateAndFillCardinalityEstimator(cardinality, bits);
CardinalityEstimatorSerializer serializer = new CardinalityEstimatorSerializer();
using (var stream = new MemoryStream())
{
serializer.Serialize(stream, estimator, true);
stream.Seek(0, SeekOrigin.Begin);
var deserializedEstimator = serializer.Deserialize(stream);
Assert.AreEqual(estimator.Count(), deserializedEstimator.Count(), "Estimators should have same count before and after serialization");
}
}
}
}
private void TestDeserializer(int cardinality)
{
CardinalityEstimator hll = CreateAndFillCardinalityEstimator(cardinality);
CardinalityEstimator hll2;
var serializer = new CardinalityEstimatorSerializer();
byte[] results;
using (var memoryStream = new MemoryStream())
{
serializer.Serialize(memoryStream, hll, false);
results = memoryStream.ToArray();
}
using (var memoryStream = new MemoryStream(results))
{
hll2 = serializer.Deserialize(memoryStream, false);
}
CompareHLL(hll, hll2);
}
public static CardinalityEstimator DeserializeCardinality(this byte[] estimator, CardinalityEstimatorSerializer serializer)
{
var stream = new MemoryStream(estimator);
return(serializer.Deserialize(stream));
}
public HyperLogLog(Stream stream)
{
_hyperLogLog = Serializer.Deserialize(stream);
}
private void TestDeserializer(int cardinality)
{
CardinalityEstimator hll = CreateAndFillCardinalityEstimator(cardinality);
CardinalityEstimator hll2;
var serializer = new CardinalityEstimatorSerializer();
byte[] results;
using (var memoryStream = new MemoryStream())
{
serializer.Serialize(memoryStream, hll);
results = memoryStream.ToArray();
}
using (var memoryStream = new MemoryStream(results))
{
hll2 = serializer.Deserialize(memoryStream);
}
CardinalityEstimatorState data = hll.GetState();
CardinalityEstimatorState data2 = hll2.GetState();
Assert.AreEqual(data.BitsPerIndex, data2.BitsPerIndex);
Assert.AreEqual(data.IsSparse, data2.IsSparse);
Assert.IsTrue((data.DirectCount != null && data2.DirectCount != null) || (data.DirectCount == null && data2.DirectCount == null));
Assert.IsTrue((data.LookupSparse != null && data2.LookupSparse != null) ||
(data.LookupSparse == null && data2.LookupSparse == null));
Assert.IsTrue((data.LookupDense != null && data2.LookupDense != null) || (data.LookupDense == null && data2.LookupDense == null));
if (data.DirectCount != null)
{
// DirectCount are subsets of each-other => they are the same set
Assert.IsTrue(data.DirectCount.IsSubsetOf(data2.DirectCount) && data2.DirectCount.IsSubsetOf(data.DirectCount));
}
if (data.LookupSparse != null)
{
Assert.IsTrue(data.LookupSparse.DictionaryEqual(data2.LookupSparse));
}
if (data.LookupDense != null)
{
Assert.IsTrue(data.LookupDense.SequenceEqual(data2.LookupDense));
}
}
public void SerializerCanDeserializeVersion1Point0()
{
var serializer = new CardinalityEstimatorSerializer();
CardinalityEstimator hllDirect = serializer.Deserialize(new MemoryStream(Resources.serializedDirect_v1_0));
CardinalityEstimator hllSparse = serializer.Deserialize(new MemoryStream(Resources.serializedSparse_v1_0));
CardinalityEstimator hllDense = serializer.Deserialize(new MemoryStream(Resources.serializedDense_v1_0));
Assert.AreEqual(50UL, hllDirect.Count());
Assert.AreEqual(151UL, hllSparse.Count());
Assert.AreEqual(5005UL, hllDense.Count());
}