Example #1
0
        /* HUF_buildTree():
         * Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree.
         *
         * @param huffNode        The array sorted by HUF_sort(). Builds the Huffman tree in this array.
         * @param maxSymbolValue  The maximum symbol value.
         * @return                The smallest node in the Huffman tree (by count).
         */
        private static int HUF_buildTree(nodeElt_s *huffNode, uint maxSymbolValue)
        {
            nodeElt_s *huffNode0 = huffNode - 1;
            int        nonNullRank;
            int        lowS, lowN;
            int        nodeNb = (255 + 1);
            int        n, nodeRoot;

            nonNullRank = (int)(maxSymbolValue);
            while (huffNode[nonNullRank].count == 0)
            {
                nonNullRank--;
            }

            lowS     = nonNullRank;
            nodeRoot = nodeNb + lowS - 1;
            lowN     = nodeNb;
            huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS - 1].count;
            huffNode[lowS].parent  = huffNode[lowS - 1].parent = (ushort)(nodeNb);
            nodeNb++;
            lowS -= 2;
            for (n = nodeNb; n <= nodeRoot; n++)
            {
                huffNode[n].count = (uint)(1U << 30);
            }

            huffNode0[0].count = (uint)(1U << 31);
            while (nodeNb <= nodeRoot)
            {
                int n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
                int n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;

                huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
                huffNode[n1].parent    = huffNode[n2].parent = (ushort)(nodeNb);
                nodeNb++;
            }

            huffNode[nodeRoot].nbBits = 0;
            for (n = nodeRoot - 1; n >= (255 + 1); n--)
            {
                huffNode[n].nbBits = (byte)(huffNode[huffNode[n].parent].nbBits + 1);
            }

            for (n = 0; n <= nonNullRank; n++)
            {
                huffNode[n].nbBits = (byte)(huffNode[huffNode[n].parent].nbBits + 1);
            }

            return(nonNullRank);
        }
Example #2
0
        public static nuint HUF_buildCTable_wksp(HUF_CElt_s *tree, uint *count, uint maxSymbolValue, uint maxNbBits, void *workSpace, nuint wkspSize)
        {
            HUF_buildCTable_wksp_tables *wksp_tables = (HUF_buildCTable_wksp_tables *)(workSpace);
            nodeElt_s *huffNode0 = (nodeElt_s *)wksp_tables->huffNodeTbl;
            nodeElt_s *huffNode  = huffNode0 + 1;
            int        nonNullRank;

            if (((nuint)(workSpace) & 3) != 0)
            {
                return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
            }

            if (wkspSize < (nuint)(sizeof(HUF_buildCTable_wksp_tables)))
            {
                return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_workSpace_tooSmall)));
            }

            if (maxNbBits == 0)
            {
                maxNbBits = 11;
            }

            if (maxSymbolValue > 255)
            {
                return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooLarge)));
            }

            memset((void *)(huffNode0), (0), ((nuint)(sizeof(nodeElt_s) * 512)));
            HUF_sort(huffNode, count, maxSymbolValue, (rankPos *)wksp_tables->rankPosition);
            nonNullRank = HUF_buildTree(huffNode, maxSymbolValue);
            maxNbBits   = HUF_setMaxHeight(huffNode, (uint)(nonNullRank), maxNbBits);
            if (maxNbBits > 12)
            {
                return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
            }

            HUF_buildCTableFromTree(tree, huffNode, nonNullRank, maxSymbolValue, maxNbBits);
            return(maxNbBits);
        }
Example #3
0
        /**
         * HUF_buildCTableFromTree():
         * Build the CTable given the Huffman tree in huffNode.
         *
         * @param[out] CTable         The output Huffman CTable.
         * @param      huffNode       The Huffman tree.
         * @param      nonNullRank    The last and smallest node in the Huffman tree.
         * @param      maxSymbolValue The maximum symbol value.
         * @param      maxNbBits      The exact maximum number of bits used in the Huffman tree.
         */
        private static void HUF_buildCTableFromTree(HUF_CElt_s *CTable, nodeElt_s *huffNode, int nonNullRank, uint maxSymbolValue, uint maxNbBits)
        {
            int     n;
            ushort *nbPerRank = stackalloc ushort[13];

            memset(nbPerRank, 0, sizeof(ushort) * 13);
            ushort *valPerRank = stackalloc ushort[13];

            memset(valPerRank, 0, sizeof(ushort) * 13);
            int alphabetSize = (int)(maxSymbolValue + 1);

            for (n = 0; n <= nonNullRank; n++)
            {
                nbPerRank[huffNode[n].nbBits]++;
            }


            {
                ushort min = 0;

                for (n = (int)(maxNbBits); n > 0; n--)
                {
                    valPerRank[n] = min;
                    min          += (ushort)(nbPerRank[n]);
                    min         >>= 1;
                }
            }

            for (n = 0; n < alphabetSize; n++)
            {
                CTable[huffNode[n].@byte].nbBits = huffNode[n].nbBits;
            }

            for (n = 0; n < alphabetSize; n++)
            {
                CTable[n].val = valPerRank[CTable[n].nbBits]++;
            }
        }
Example #4
0
        /**
         * HUF_sort():
         * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order.
         *
         * @param[out] huffNode       Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled.
         *                            Must have (maxSymbolValue + 1) entries.
         * @param[in]  count          Histogram of the symbols.
         * @param[in]  maxSymbolValue Maximum symbol value.
         * @param      rankPosition   This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries.
         */
        private static void HUF_sort(nodeElt_s *huffNode, uint *count, uint maxSymbolValue, rankPos *rankPosition)
        {
            int n;
            int maxSymbolValue1 = (int)(maxSymbolValue) + 1;

            memset((void *)(rankPosition), (0), ((nuint)(sizeof(rankPos)) * 32));
            for (n = 0; n < maxSymbolValue1; ++n)
            {
                uint lowerRank = BIT_highbit32(count[n] + 1);

                rankPosition[lowerRank].@base++;
            }

            assert(rankPosition[32 - 1].@base == 0);
            for (n = 32 - 1; n > 0; --n)
            {
                rankPosition[n - 1].@base += rankPosition[n].@base;
                rankPosition[n - 1].curr   = rankPosition[n - 1].@base;
            }

            for (n = 0; n < maxSymbolValue1; ++n)
            {
                uint c   = count[n];
                uint r   = BIT_highbit32(c + 1) + 1;
                uint pos = rankPosition[r].curr++;

                while ((pos > rankPosition[r].@base) && (c > huffNode[pos - 1].count))
                {
                    huffNode[pos] = huffNode[pos - 1];
                    pos--;
                }

                huffNode[pos].count = c;
                huffNode[pos].@byte = (byte)(n);
            }
        }
Example #5
0
        /**
         * HUF_setMaxHeight():
         * Enforces maxNbBits on the Huffman tree described in huffNode.
         *
         * It sets all nodes with nbBits > maxNbBits to be maxNbBits. Then it adjusts
         * the tree to so that it is a valid canonical Huffman tree.
         *
         * @pre               The sum of the ranks of each symbol == 2^largestBits,
         *                    where largestBits == huffNode[lastNonNull].nbBits.
         * @post              The sum of the ranks of each symbol == 2^largestBits,
         *                    where largestBits is the return value <= maxNbBits.
         *
         * @param huffNode    The Huffman tree modified in place to enforce maxNbBits.
         * @param lastNonNull The symbol with the lowest count in the Huffman tree.
         * @param maxNbBits   The maximum allowed number of bits, which the Huffman tree
         *                    may not respect. After this function the Huffman tree will
         *                    respect maxNbBits.
         * @return            The maximum number of bits of the Huffman tree after adjustment,
         *                    necessarily no more than maxNbBits.
         */
        private static uint HUF_setMaxHeight(nodeElt_s *huffNode, uint lastNonNull, uint maxNbBits)
        {
            uint largestBits = huffNode[lastNonNull].nbBits;

            if (largestBits <= maxNbBits)
            {
                return(largestBits);
            }


            {
                int  totalCost = 0;
                uint baseCost  = (uint)(1 << (int)(largestBits - maxNbBits));
                int  n         = (int)(lastNonNull);

                while (huffNode[n].nbBits > maxNbBits)
                {
                    totalCost         += (int)((int)(baseCost - (uint)((1 << (int)(largestBits - huffNode[n].nbBits)))));
                    huffNode[n].nbBits = (byte)(maxNbBits);
                    n--;
                }

                assert(huffNode[n].nbBits <= maxNbBits);
                while (huffNode[n].nbBits == maxNbBits)
                {
                    --n;
                }

                assert(((uint)totalCost & (baseCost - 1)) == 0);
                totalCost >>= (int)(largestBits - maxNbBits);
                assert(totalCost > 0);

                {
                    uint  noSymbol = 0xF0F0F0F0;
                    uint *rankLast = stackalloc uint[14];

                    memset((void *)(rankLast), (0xF0), ((nuint)(sizeof(uint) * 14)));

                    {
                        uint currentNbBits = maxNbBits;
                        int  pos;

                        for (pos = n; pos >= 0; pos--)
                        {
                            if (huffNode[pos].nbBits >= currentNbBits)
                            {
                                continue;
                            }

                            currentNbBits = huffNode[pos].nbBits;
                            rankLast[maxNbBits - currentNbBits] = (uint)(pos);
                        }
                    }

                    while (totalCost > 0)
                    {
                        uint nBitsToDecrease = BIT_highbit32((uint)(totalCost)) + 1;

                        for (; nBitsToDecrease > 1; nBitsToDecrease--)
                        {
                            uint highPos = rankLast[nBitsToDecrease];
                            uint lowPos  = rankLast[nBitsToDecrease - 1];

                            if (highPos == noSymbol)
                            {
                                continue;
                            }

                            if (lowPos == noSymbol)
                            {
                                break;
                            }


                            {
                                uint highTotal = huffNode[highPos].count;
                                uint lowTotal  = 2 * huffNode[lowPos].count;

                                if (highTotal <= lowTotal)
                                {
                                    break;
                                }
                            }
                        }

                        assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1);
                        while ((nBitsToDecrease <= 12) && (rankLast[nBitsToDecrease] == noSymbol))
                        {
                            nBitsToDecrease++;
                        }

                        assert(rankLast[nBitsToDecrease] != noSymbol);
                        totalCost -= 1 << (int)(nBitsToDecrease - 1);
                        huffNode[rankLast[nBitsToDecrease]].nbBits++;
                        if (rankLast[nBitsToDecrease - 1] == noSymbol)
                        {
                            rankLast[nBitsToDecrease - 1] = rankLast[nBitsToDecrease];
                        }

                        if (rankLast[nBitsToDecrease] == 0)
                        {
                            rankLast[nBitsToDecrease] = noSymbol;
                        }
                        else
                        {
                            rankLast[nBitsToDecrease]--;
                            if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits - nBitsToDecrease)
                            {
                                rankLast[nBitsToDecrease] = noSymbol;
                            }
                        }
                    }

                    while (totalCost < 0)
                    {
                        if (rankLast[1] == noSymbol)
                        {
                            while (huffNode[n].nbBits == maxNbBits)
                            {
                                n--;
                            }

                            huffNode[n + 1].nbBits--;
                            assert(n >= 0);
                            rankLast[1] = (uint)(n + 1);
                            totalCost++;
                            continue;
                        }

                        huffNode[rankLast[1] + 1].nbBits--;
                        rankLast[1]++;
                        totalCost++;
                    }
                }
            }

            return(maxNbBits);
        }