Exemple #1
0
        /// <summary>
        ///     Deserialize a <see cref="CardinalityEstimator" /> from the given <paramref name="stream" />
        /// </summary>
        public CardinalityEstimator Deserialize(Stream stream)
        {
            using (var br = new BinaryReader(stream))
            {
                int dataFormatMajorVersion = br.ReadUInt16();
                int dataFormatMinorVersion = br.ReadUInt16();

                AssertDataVersionCanBeRead(dataFormatMajorVersion, dataFormatMinorVersion);

                int  bitsPerIndex  = br.ReadInt32();
                byte flags         = br.ReadByte();
                bool isSparse      = ((flags & 2) == 2);
                bool isDirectCount = ((flags & 1) == 1);

                HashSet <ulong>            directCount  = null;
                IDictionary <ushort, byte> lookupSparse = isSparse ? new Dictionary <ushort, byte>() : null;
                byte[] lookupDense = null;

                if (isDirectCount)
                {
                    int count = br.ReadInt32();
                    directCount = new HashSet <ulong>();

                    for (var i = 0; i < count; i++)
                    {
                        ulong element = br.ReadUInt64();
                        directCount.Add(element);
                    }
                }
                else if (isSparse)
                {
                    int count = br.ReadInt32();

                    for (var i = 0; i < count; i++)
                    {
                        ushort elementKey   = br.ReadUInt16();
                        byte   elementValue = br.ReadByte();
                        lookupSparse.Add(elementKey, elementValue);
                    }
                }
                else
                {
                    int count = br.ReadInt32();
                    lookupDense = br.ReadBytes(count);
                }

                var data = new CardinalityEstimatorState
                {
                    BitsPerIndex = bitsPerIndex,
                    DirectCount  = directCount,
                    IsSparse     = isSparse,
                    LookupDense  = lookupDense,
                    LookupSparse = lookupSparse
                };

                var result = new CardinalityEstimator(data);

                return(result);
            }
        }
        /// <summary>
        ///     Creates a CardinalityEstimator with the given <paramref name="state" />
        /// </summary>
        internal CardinalityEstimator(CardinalityEstimatorState state)
        {
            this.bitsPerIndex = state.BitsPerIndex;
            this.bitsForHll   = 64 - this.bitsPerIndex;
            this.m            = (int)Math.Pow(2, this.bitsPerIndex);
            this.alphaM       = GetAlphaM(this.m);
            this.subAlgorithmSelectionThreshold = GetSubAlgorithmSelectionThreshold(this.bitsPerIndex);

            // Init the hash function
            this.hashFunctionId = state.HashFunctionId;
            this.hashFunction   = HashFunctionFactory.GetHashFunction(this.hashFunctionId);

            // Init the direct count
            this.directCount = state.DirectCount != null ? new HashSet <ulong>(state.DirectCount) : null;

            // Init the sparse representation
            this.isSparse       = state.IsSparse;
            this.lookupSparse   = state.LookupSparse != null ? new Dictionary <ushort, byte>(state.LookupSparse) : null;
            this.lookupDense    = state.LookupDense;
            this.CountAdditions = state.CountAdditions;

            // Each element in the sparse representation takes 15 bytes, and there is some constant overhead
            this.sparseMaxElements = Math.Max(0, this.m / 15 - 10);
            // If necessary, switch to the dense representation
            if (this.sparseMaxElements <= 0)
            {
                SwitchToDenseRepresentation();
            }

            // if DirectCount is not null, populate the HLL lookup with its elements.  This allows serialization to include only directCount
            if (this.directCount != null)
            {
                // since we are re-initializing the object, we need to reset isSparse to true and sparse lookup
                isSparse          = true;
                this.lookupSparse = new Dictionary <ushort, byte>();
                foreach (ulong element in this.directCount)
                {
                    AddElementHash(element);
                }
            }
            else
            {
                this.directCount = null;
            }
        }
Exemple #3
0
        /// <summary>
        ///     Serialize the given <paramref name="cardinalityEstimator" /> to <paramref name="stream" />
        /// </summary>
        public void Serialize(Stream stream, CardinalityEstimator cardinalityEstimator)
        {
            using (var bw = new BinaryWriter(stream))
            {
                bw.Write(DataFormatMajorVersion);
                bw.Write(DataFormatMinorVersion);

                CardinalityEstimatorState data = cardinalityEstimator.GetState();

                bw.Write((byte)data.HashFunctionId);
                bw.Write(data.BitsPerIndex);
                bw.Write((byte)(((data.IsSparse ? 1 : 0) << 1) + (data.DirectCount != null ? 1 : 0)));
                if (data.DirectCount != null)
                {
                    bw.Write(data.DirectCount.Count);
                    foreach (ulong element in data.DirectCount)
                    {
                        bw.Write(element);
                    }
                }
                else if (data.IsSparse)
                {
                    bw.Write(data.LookupSparse.Count);
                    foreach (KeyValuePair <ushort, byte> element in data.LookupSparse)
                    {
                        bw.Write(element.Key);
                        bw.Write(element.Value);
                    }
                }
                else
                {
                    bw.Write(data.LookupDense.Length);
                    foreach (byte element in data.LookupDense)
                    {
                        bw.Write(element);
                    }
                }

                bw.Write(data.CountAdditions);
                bw.Flush();
            }
        }
Exemple #4
0
        /// <summary>
        ///     Deserialize a <see cref="CardinalityEstimator" /> from the given <paramref name="stream" />
        /// </summary>
        public CardinalityEstimator Deserialize(Stream stream)
        {
            using (var br = new BinaryReader(stream))
            {
                int dataFormatMajorVersion = br.ReadUInt16();
                int dataFormatMinorVersion = br.ReadUInt16();

                AssertDataVersionCanBeRead(dataFormatMajorVersion, dataFormatMinorVersion);

                HashFunctionId hashFunctionId;
                if (dataFormatMajorVersion >= 2)
                {
                    // Starting with version 2.0, the serializer writes the hash function ID
                    hashFunctionId = (HashFunctionId)br.ReadByte();
                }
                else
                {
                    // Versions before 2.0 all used FNV-1a
                    hashFunctionId = HashFunctionId.Fnv1A;
                }

                int  bitsPerIndex  = br.ReadInt32();
                byte flags         = br.ReadByte();
                bool isSparse      = ((flags & 2) == 2);
                bool isDirectCount = ((flags & 1) == 1);

                HashSet <ulong>            directCount  = null;
                IDictionary <ushort, byte> lookupSparse = isSparse ? new Dictionary <ushort, byte>() : null;
                byte[] lookupDense = null;

                if (isDirectCount)
                {
                    int count = br.ReadInt32();
                    directCount = new HashSet <ulong>();

                    for (var i = 0; i < count; i++)
                    {
                        ulong element = br.ReadUInt64();
                        directCount.Add(element);
                    }
                }
                else if (isSparse)
                {
                    int count = br.ReadInt32();

                    for (var i = 0; i < count; i++)
                    {
                        ushort elementKey   = br.ReadUInt16();
                        byte   elementValue = br.ReadByte();
                        lookupSparse.Add(elementKey, elementValue);
                    }
                }
                else
                {
                    int count = br.ReadInt32();
                    lookupDense = br.ReadBytes(count);
                }

                // Starting with version 2.1, the serializer writes CountAdditions
                ulong countAdditions = 0UL;
                if (dataFormatMajorVersion >= 2 && dataFormatMinorVersion >= 1)
                {
                    countAdditions = br.ReadUInt64();
                }

                var data = new CardinalityEstimatorState
                {
                    HashFunctionId = hashFunctionId,
                    BitsPerIndex   = bitsPerIndex,
                    DirectCount    = directCount,
                    IsSparse       = isSparse,
                    LookupDense    = lookupDense,
                    LookupSparse   = lookupSparse,
                    CountAdditions = countAdditions,
                };

                var result = new CardinalityEstimator(data);

                return(result);
            }
        }
        /// <summary>
        ///     Deserialize a <see cref="CardinalityEstimator" /> from the given <paramref name="stream" />
        /// </summary>
        public CardinalityEstimator Deserialize(Stream stream)
        {
            using (var br = new BinaryReader(stream))
            {
                int dataFormatMajorVersion = br.ReadUInt16();
                int dataFormatMinorVersion = br.ReadUInt16();

                AssertDataVersionCanBeRead(dataFormatMajorVersion, dataFormatMinorVersion);

                int bitsPerIndex = br.ReadInt32();
                byte flags = br.ReadByte();
                bool isSparse = ((flags & 2) == 2);
                bool isDirectCount = ((flags & 1) == 1);

                HashSet<ulong> directCount = null;
                IDictionary<ushort, byte> lookupSparse = isSparse ? new Dictionary<ushort, byte>() : null;
                byte[] lookupDense = null;

                if (isDirectCount)
                {
                    int count = br.ReadInt32();
                    directCount = new HashSet<ulong>();

                    for (var i = 0; i < count; i++)
                    {
                        ulong element = br.ReadUInt64();
                        directCount.Add(element);
                    }
                }
                else if (isSparse)
                {
                    int count = br.ReadInt32();

                    for (var i = 0; i < count; i++)
                    {
                        ushort elementKey = br.ReadUInt16();
                        byte elementValue = br.ReadByte();
                        lookupSparse.Add(elementKey, elementValue);
                    }
                }
                else
                {
                    int count = br.ReadInt32();
                    lookupDense = br.ReadBytes(count);
                }

                var data = new CardinalityEstimatorState
                {
                    BitsPerIndex = bitsPerIndex,
                    DirectCount = directCount,
                    IsSparse = isSparse,
                    LookupDense = lookupDense,
                    LookupSparse = lookupSparse
                };

                var result = new CardinalityEstimator(data);

                return result;
            }
        }