private static unsafe void InitHuffmanTree(HuffmanTree *self, uint count,
short left, short right)
{
self->total_count_ = count;
self->index_left_ = left;
self->index_right_or_value_ = right;
}
/* Sort the root nodes, least popular first. */
private static unsafe bool SortHuffmanTreeEntropyEncode(
HuffmanTree *v0, HuffmanTree *v1)
{
if (v0->total_count_ != v1->total_count_)
{
return(v0->total_count_ < v1->total_count_);
}
return(v0->index_right_or_value_ > v1->index_right_or_value_);
}
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
* "bits" based on "histogram" and stores it into the bit stream. */
private static unsafe void BuildAndStoreCommandPrefixCode(
uint *histogram,
byte *depth, ushort *bits,
size_t *storage_ix, byte *storage)
{
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
HuffmanTree *tree = stackalloc HuffmanTree[129];
byte * cmd_depth = stackalloc byte[BROTLI_NUM_COMMAND_SYMBOLS];
memset(cmd_depth, 0, BROTLI_NUM_COMMAND_SYMBOLS * sizeof(byte));
ushort *cmd_bits = stackalloc ushort[64];
BrotliCreateHuffmanTree(histogram, 64, 15, tree, depth);
BrotliCreateHuffmanTree(&histogram[64], 64, 14, tree, &depth[64]);
/* We have to jump through a few hoops here in order to compute
* the command bits because the symbols are in a different order than in
* the full alphabet. This looks complicated, but having the symbols
* in this order in the command bits saves a few branches in the Emit*
* functions. */
memcpy(cmd_depth, depth + 24, 24);
memcpy(cmd_depth + 24, depth, 8);
memcpy(cmd_depth + 32, depth + 48, 8);
memcpy(cmd_depth + 40, depth + 8, 8);
memcpy(cmd_depth + 48, depth + 56, 8);
memcpy(cmd_depth + 56, depth + 16, 8);
BrotliConvertBitDepthsToSymbols(cmd_depth, 64, cmd_bits);
memcpy(bits, cmd_bits + 24, 16);
memcpy(bits + 8, cmd_bits + 40, 16);
memcpy(bits + 16, cmd_bits + 56, 16);
memcpy(bits + 24, cmd_bits, 48);
memcpy(bits + 48, cmd_bits + 32, 16);
memcpy(bits + 56, cmd_bits + 48, 16);
BrotliConvertBitDepthsToSymbols(&depth[64], 64, &bits[64]);
{
/* Create the bit length array for the full command alphabet. */
size_t i;
memset(cmd_depth, 0, 64); /* only 64 first values were used */
memcpy(cmd_depth, depth + 24, 8);
memcpy(cmd_depth + 64, depth + 32, 8);
memcpy(cmd_depth + 128, depth + 40, 8);
memcpy(cmd_depth + 192, depth + 48, 8);
memcpy(cmd_depth + 384, depth + 56, 8);
for (i = 0; i < 8; ++i)
{
cmd_depth[128 + 8 * i] = depth[i];
cmd_depth[256 + 8 * i] = depth[8 + i];
cmd_depth[448 + 8 * i] = depth[16 + i];
}
BrotliStoreHuffmanTree(
cmd_depth, BROTLI_NUM_COMMAND_SYMBOLS, tree, storage_ix, storage);
}
BrotliStoreHuffmanTree(&depth[64], 64, tree, storage_ix, storage);
}
private static unsafe void SortHuffmanTreeItems(HuffmanTree *items, size_t n,
HuffmanTreeComparator comparator)
{
size_t[] gaps = { 132, 57, 23, 10, 4, 1 };
if (n < 13)
{
/* Insertion sort. */
size_t i;
for (i = 1; i < n; ++i)
{
HuffmanTree tmp = items[i];
size_t k = i;
size_t j = i - 1;
while (comparator(&tmp, &items[j]))
{
items[k] = items[j];
k = j;
if (j-- == 0)
{
break;
}
}
items[k] = tmp;
}
}
else
{
/* Shell sort. */
int g = n < 57 ? 2 : 0;
for (; g < 6; ++g)
{
size_t gap = gaps[g];
size_t i;
for (i = gap; i < n; ++i)
{
size_t j = i;
HuffmanTree tmp = items[i];
for (; j >= gap && comparator(&tmp, &items[j - gap]); j -= gap)
{
items[j] = items[j - gap];
}
items[j] = tmp;
}
}
}
}
public static unsafe void BuildAndStoreEntropyCodes(ref MemoryManager m, BlockEncoder *self,
HistogramLiteral *histograms, size_t histograms_size,
HuffmanTree *tree, size_t *storage_ix, byte *storage)
{
size_t alphabet_size = self->alphabet_size_;
size_t table_size = histograms_size * alphabet_size;
self->depths_ = (byte *)BrotliAllocate(ref m, table_size * sizeof(byte));
self->bits_ = (ushort *)BrotliAllocate(ref m, table_size * sizeof(ushort));
{
size_t i;
for (i = 0; i < histograms_size; ++i)
{
size_t ix = i * alphabet_size;
BuildAndStoreHuffmanTree(&histograms[i].data_[0], alphabet_size, tree,
&self->depths_[ix], &self->bits_[ix], storage_ix, storage);
}
}
}
private static unsafe bool BrotliSetDepth(
int p0, HuffmanTree *pool, byte *depth, int max_depth)
{
int[] stack = new int[16];
int level = 0;
int p = p0;
stack[0] = -1;
while (true)
{
if (pool[p].index_left_ >= 0)
{
level++;
if (level > max_depth)
{
return(false);
}
stack[level] = pool[p].index_right_or_value_;
p = pool[p].index_left_;
continue;
}
else
{
depth[pool[p].index_right_or_value_] = (byte)level;
}
while (level >= 0 && stack[level] == -1)
{
level--;
}
if (level < 0)
{
return(true);
}
p = stack[level];
stack[level] = -1;
}
}
/* This function will create a Huffman tree.
*
* The catch here is that the tree cannot be arbitrarily deep.
* Brotli specifies a maximum depth of 15 bits for "code trees"
* and 7 bits for "code length code trees."
*
* count_limit is the value that is to be faked as the minimum value
* and this minimum value is raised until the tree matches the
* maximum length requirement.
*
* This algorithm is not of excellent performance for very long data blocks,
* especially when population counts are longer than 2**tree_limit, but
* we are not planning to use this with extremely long blocks.
*
* See http://en.wikipedia.org/wiki/Huffman_coding */
private static unsafe void BrotliCreateHuffmanTree(uint *data,
size_t length,
int tree_limit,
HuffmanTree *tree,
byte *depth)
{
uint count_limit;
HuffmanTree sentinel;
InitHuffmanTree(&sentinel, uint.MaxValue, -1, -1);
/* For block sizes below 64 kB, we never need to do a second iteration
* of this loop. Probably all of our block sizes will be smaller than
* that, so this loop is mostly of academic interest. If we actually
* would need this, we would be better off with the Katajainen algorithm. */
for (count_limit = 1;; count_limit *= 2)
{
size_t n = 0;
size_t i;
size_t j;
size_t k;
for (i = length; i != 0;)
{
--i;
if (data[i] != 0)
{
uint count = Math.Max(data[i], count_limit);
InitHuffmanTree(&tree[n++], count, -1, (short)i);
}
}
if (n == 1)
{
depth[tree[0].index_right_or_value_] = 1; /* Only one element. */
break;
}
SortHuffmanTreeItems(tree, n, SortHuffmanTreeEntropyEncode);
/* The nodes are:
* [0, n): the sorted leaf nodes that we start with.
* [n]: we add a sentinel here.
* [n + 1, 2n): new parent nodes are added here, starting from
* (n+1). These are naturally in ascending order.
* [2n]: we add a sentinel at the end as well.
* There will be (2n+1) elements at the end. */
tree[n] = sentinel;
tree[n + 1] = sentinel;
i = 0; /* Points to the next leaf node. */
j = n + 1; /* Points to the next non-leaf node. */
for (k = n - 1; k != 0; --k)
{
size_t left, right;
if (tree[i].total_count_ <= tree[j].total_count_)
{
left = i;
++i;
}
else
{
left = j;
++j;
}
if (tree[i].total_count_ <= tree[j].total_count_)
{
right = i;
++i;
}
else
{
right = j;
++j;
}
{
/* The sentinel node becomes the parent node. */
size_t j_end = 2 * n - k;
tree[j_end].total_count_ =
tree[left].total_count_ + tree[right].total_count_;
tree[j_end].index_left_ = (short)left;
tree[j_end].index_right_or_value_ = (short)right;
/* Add back the last sentinel node. */
tree[j_end + 1] = sentinel;
}
}
if (BrotliSetDepth((int)(2 * n - 1), &tree[0], depth, tree_limit))
{
/* We need to pack the Huffman tree in tree_limit bits. If this was not
* successful, add fake entities to the lowest values and retry. */
break;
}
}
}