private static unsafe void InitHuffmanTree(HuffmanTree *self, uint count, short left, short right) { self->total_count_ = count; self->index_left_ = left; self->index_right_or_value_ = right; }
/* Sort the root nodes, least popular first. */ private static unsafe bool SortHuffmanTreeEntropyEncode( HuffmanTree *v0, HuffmanTree *v1) { if (v0->total_count_ != v1->total_count_) { return(v0->total_count_ < v1->total_count_); } return(v0->index_right_or_value_ > v1->index_right_or_value_); }
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and * "bits" based on "histogram" and stores it into the bit stream. */ private static unsafe void BuildAndStoreCommandPrefixCode( uint *histogram, byte *depth, ushort *bits, size_t *storage_ix, byte *storage) { /* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */ HuffmanTree *tree = stackalloc HuffmanTree[129]; byte * cmd_depth = stackalloc byte[BROTLI_NUM_COMMAND_SYMBOLS]; memset(cmd_depth, 0, BROTLI_NUM_COMMAND_SYMBOLS * sizeof(byte)); ushort *cmd_bits = stackalloc ushort[64]; BrotliCreateHuffmanTree(histogram, 64, 15, tree, depth); BrotliCreateHuffmanTree(&histogram[64], 64, 14, tree, &depth[64]); /* We have to jump through a few hoops here in order to compute * the command bits because the symbols are in a different order than in * the full alphabet. This looks complicated, but having the symbols * in this order in the command bits saves a few branches in the Emit* * functions. */ memcpy(cmd_depth, depth + 24, 24); memcpy(cmd_depth + 24, depth, 8); memcpy(cmd_depth + 32, depth + 48, 8); memcpy(cmd_depth + 40, depth + 8, 8); memcpy(cmd_depth + 48, depth + 56, 8); memcpy(cmd_depth + 56, depth + 16, 8); BrotliConvertBitDepthsToSymbols(cmd_depth, 64, cmd_bits); memcpy(bits, cmd_bits + 24, 16); memcpy(bits + 8, cmd_bits + 40, 16); memcpy(bits + 16, cmd_bits + 56, 16); memcpy(bits + 24, cmd_bits, 48); memcpy(bits + 48, cmd_bits + 32, 16); memcpy(bits + 56, cmd_bits + 48, 16); BrotliConvertBitDepthsToSymbols(&depth[64], 64, &bits[64]); { /* Create the bit length array for the full command alphabet. */ size_t i; memset(cmd_depth, 0, 64); /* only 64 first values were used */ memcpy(cmd_depth, depth + 24, 8); memcpy(cmd_depth + 64, depth + 32, 8); memcpy(cmd_depth + 128, depth + 40, 8); memcpy(cmd_depth + 192, depth + 48, 8); memcpy(cmd_depth + 384, depth + 56, 8); for (i = 0; i < 8; ++i) { cmd_depth[128 + 8 * i] = depth[i]; cmd_depth[256 + 8 * i] = depth[8 + i]; cmd_depth[448 + 8 * i] = depth[16 + i]; } BrotliStoreHuffmanTree( cmd_depth, BROTLI_NUM_COMMAND_SYMBOLS, tree, storage_ix, storage); } BrotliStoreHuffmanTree(&depth[64], 64, tree, storage_ix, storage); }
private static unsafe void SortHuffmanTreeItems(HuffmanTree *items, size_t n, HuffmanTreeComparator comparator) { size_t[] gaps = { 132, 57, 23, 10, 4, 1 }; if (n < 13) { /* Insertion sort. */ size_t i; for (i = 1; i < n; ++i) { HuffmanTree tmp = items[i]; size_t k = i; size_t j = i - 1; while (comparator(&tmp, &items[j])) { items[k] = items[j]; k = j; if (j-- == 0) { break; } } items[k] = tmp; } } else { /* Shell sort. */ int g = n < 57 ? 2 : 0; for (; g < 6; ++g) { size_t gap = gaps[g]; size_t i; for (i = gap; i < n; ++i) { size_t j = i; HuffmanTree tmp = items[i]; for (; j >= gap && comparator(&tmp, &items[j - gap]); j -= gap) { items[j] = items[j - gap]; } items[j] = tmp; } } } }
public static unsafe void BuildAndStoreEntropyCodes(ref MemoryManager m, BlockEncoder *self, HistogramLiteral *histograms, size_t histograms_size, HuffmanTree *tree, size_t *storage_ix, byte *storage) { size_t alphabet_size = self->alphabet_size_; size_t table_size = histograms_size * alphabet_size; self->depths_ = (byte *)BrotliAllocate(ref m, table_size * sizeof(byte)); self->bits_ = (ushort *)BrotliAllocate(ref m, table_size * sizeof(ushort)); { size_t i; for (i = 0; i < histograms_size; ++i) { size_t ix = i * alphabet_size; BuildAndStoreHuffmanTree(&histograms[i].data_[0], alphabet_size, tree, &self->depths_[ix], &self->bits_[ix], storage_ix, storage); } } }
private static unsafe bool BrotliSetDepth( int p0, HuffmanTree *pool, byte *depth, int max_depth) { int[] stack = new int[16]; int level = 0; int p = p0; stack[0] = -1; while (true) { if (pool[p].index_left_ >= 0) { level++; if (level > max_depth) { return(false); } stack[level] = pool[p].index_right_or_value_; p = pool[p].index_left_; continue; } else { depth[pool[p].index_right_or_value_] = (byte)level; } while (level >= 0 && stack[level] == -1) { level--; } if (level < 0) { return(true); } p = stack[level]; stack[level] = -1; } }
/* This function will create a Huffman tree. * * The catch here is that the tree cannot be arbitrarily deep. * Brotli specifies a maximum depth of 15 bits for "code trees" * and 7 bits for "code length code trees." * * count_limit is the value that is to be faked as the minimum value * and this minimum value is raised until the tree matches the * maximum length requirement. * * This algorithm is not of excellent performance for very long data blocks, * especially when population counts are longer than 2**tree_limit, but * we are not planning to use this with extremely long blocks. * * See http://en.wikipedia.org/wiki/Huffman_coding */ private static unsafe void BrotliCreateHuffmanTree(uint *data, size_t length, int tree_limit, HuffmanTree *tree, byte *depth) { uint count_limit; HuffmanTree sentinel; InitHuffmanTree(&sentinel, uint.MaxValue, -1, -1); /* For block sizes below 64 kB, we never need to do a second iteration * of this loop. Probably all of our block sizes will be smaller than * that, so this loop is mostly of academic interest. If we actually * would need this, we would be better off with the Katajainen algorithm. */ for (count_limit = 1;; count_limit *= 2) { size_t n = 0; size_t i; size_t j; size_t k; for (i = length; i != 0;) { --i; if (data[i] != 0) { uint count = Math.Max(data[i], count_limit); InitHuffmanTree(&tree[n++], count, -1, (short)i); } } if (n == 1) { depth[tree[0].index_right_or_value_] = 1; /* Only one element. */ break; } SortHuffmanTreeItems(tree, n, SortHuffmanTreeEntropyEncode); /* The nodes are: * [0, n): the sorted leaf nodes that we start with. * [n]: we add a sentinel here. * [n + 1, 2n): new parent nodes are added here, starting from * (n+1). These are naturally in ascending order. * [2n]: we add a sentinel at the end as well. * There will be (2n+1) elements at the end. */ tree[n] = sentinel; tree[n + 1] = sentinel; i = 0; /* Points to the next leaf node. */ j = n + 1; /* Points to the next non-leaf node. */ for (k = n - 1; k != 0; --k) { size_t left, right; if (tree[i].total_count_ <= tree[j].total_count_) { left = i; ++i; } else { left = j; ++j; } if (tree[i].total_count_ <= tree[j].total_count_) { right = i; ++i; } else { right = j; ++j; } { /* The sentinel node becomes the parent node. */ size_t j_end = 2 * n - k; tree[j_end].total_count_ = tree[left].total_count_ + tree[right].total_count_; tree[j_end].index_left_ = (short)left; tree[j_end].index_right_or_value_ = (short)right; /* Add back the last sentinel node. */ tree[j_end + 1] = sentinel; } } if (BrotliSetDepth((int)(2 * n - 1), &tree[0], depth, tree_limit)) { /* We need to pack the Huffman tree in tree_limit bits. If this was not * successful, add fake entities to the lowest values and retry. */ break; } } }