public static unsafe void InitBlockSplitter( ref MemoryManager m, BlockSplitterDistance *self, size_t alphabet_size, size_t min_block_size, double split_threshold, size_t num_symbols, BlockSplit *split, HistogramDistance **histograms, size_t *histograms_size) { size_t max_num_blocks = num_symbols / min_block_size + 1; /* We have to allocate one more histogram than the maximum number of block * types for the current histogram when the meta-block is too big. */ size_t max_num_types = Math.Min(max_num_blocks, BROTLI_MAX_NUMBER_OF_BLOCK_TYPES + 1); self->alphabet_size_ = alphabet_size; self->min_block_size_ = min_block_size; self->split_threshold_ = split_threshold; self->num_blocks_ = 0; self->split_ = split; self->histograms_size_ = histograms_size; self->target_block_size_ = min_block_size; self->block_size_ = 0; self->curr_histogram_ix_ = 0; self->merge_last_count_ = 0; BrotliEnsureCapacity(ref m, sizeof(byte), (void **)&split->types, &split->types_alloc_size, max_num_blocks); BrotliEnsureCapacity(ref m, sizeof(uint), (void **)&split->lengths, &split->lengths_alloc_size, max_num_blocks); self->split_->num_blocks = max_num_blocks; *histograms_size = max_num_types; *histograms = (HistogramDistance *)BrotliAllocate(ref m, *histograms_size * sizeof(HistogramDistance)); self->histograms_ = *histograms; /* Clear only current histogram. */ HistogramDistance.HistogramClear(&self->histograms_[0]); self->last_histogram_ix_0 = self->last_histogram_ix_1 = 0; }
private static unsafe void InitBlockSplitIterator(BlockSplitIterator *self, BlockSplit *split) { self->split_ = split; self->idx_ = 0; self->type_ = 0; self->length_ = split->lengths != null ? split->lengths[0] : 0; }
private static unsafe void BrotliInitBlockSplit(BlockSplit *self) { self->num_types = 0; self->num_blocks = 0; self->types = null; self->lengths = null; self->types_alloc_size = 0; self->lengths_alloc_size = 0; }
private static unsafe void BrotliBuildHistogramsWithContext( Command *cmds, size_t num_commands, BlockSplit *literal_split, BlockSplit *insert_and_copy_split, BlockSplit *dist_split, byte *ringbuffer, size_t start_pos, size_t mask, byte prev_byte, byte prev_byte2, ContextType *context_modes, HistogramLiteral *literal_histograms, HistogramCommand *insert_and_copy_histograms, HistogramDistance *copy_dist_histograms) { size_t pos = start_pos; BlockSplitIterator literal_it; BlockSplitIterator insert_and_copy_it; BlockSplitIterator dist_it; size_t i; InitBlockSplitIterator(&literal_it, literal_split); InitBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split); InitBlockSplitIterator(&dist_it, dist_split); for (i = 0; i < num_commands; ++i) { Command *cmd = &cmds[i]; size_t j; BlockSplitIteratorNext(&insert_and_copy_it); HistogramCommand.HistogramAdd(&insert_and_copy_histograms[insert_and_copy_it.type_], cmd->cmd_prefix_); for (j = cmd->insert_len_; j != 0; --j) { size_t context; BlockSplitIteratorNext(&literal_it); context = context_modes != null ? ((literal_it.type_ << BROTLI_LITERAL_CONTEXT_BITS) + Context(prev_byte, prev_byte2, context_modes[literal_it.type_])) : literal_it.type_; HistogramLiteral.HistogramAdd(&literal_histograms[context], ringbuffer[pos & mask]); prev_byte2 = prev_byte; prev_byte = ringbuffer[pos & mask]; ++pos; } pos += CommandCopyLen(cmd); if (CommandCopyLen(cmd) != 0) { prev_byte2 = ringbuffer[(pos - 2) & mask]; prev_byte = ringbuffer[(pos - 1) & mask]; if (cmd->cmd_prefix_ >= 128) { size_t context; BlockSplitIteratorNext(&dist_it); context = (dist_it.type_ << BROTLI_DISTANCE_CONTEXT_BITS) + CommandDistanceContext(cmd); HistogramDistance.HistogramAdd(©_dist_histograms[context], cmd->dist_prefix_); } } } }
/* Does either of three things: * (1) emits the current block with a new block type; * (2) emits the current block with the type of the second last block; * (3) merges the current block with the last block. */ private static unsafe void ContextBlockSplitterFinishBlock( ContextBlockSplitter *self, ref MemoryManager m, bool is_final) { BlockSplit * split = self->split_; size_t num_contexts = self->num_contexts_; double * last_entropy = self->last_entropy_; HistogramLiteral *histograms = self->histograms_; if (self->block_size_ < self->min_block_size_) { self->block_size_ = self->min_block_size_; } if (self->num_blocks_ == 0) { size_t i; /* Create first block. */ split->lengths[0] = (uint)self->block_size_; split->types[0] = 0; for (i = 0; i < num_contexts; ++i) { last_entropy[i] = BitsEntropy(histograms[i].data_, self->alphabet_size_); last_entropy[num_contexts + i] = last_entropy[i]; } ++self->num_blocks_; ++split->num_types; self->curr_histogram_ix_ += num_contexts; if (self->curr_histogram_ix_ < *self->histograms_size_) { HistogramLiteral.ClearHistograms( &self->histograms_[self->curr_histogram_ix_], self->num_contexts_); } self->block_size_ = 0; } else if (self->block_size_ > 0) { /* Try merging the set of histograms for the current block type with the * respective set of histograms for the last and second last block types. * Decide over the split based on the total reduction of entropy across * all contexts. */ double * entropy = stackalloc double[BROTLI_MAX_STATIC_CONTEXTS]; HistogramLiteral *combined_histo = (HistogramLiteral *)BrotliAllocate(ref m, 2 * num_contexts * sizeof(HistogramLiteral)); double[] combined_entropy = new double[2 * BROTLI_MAX_STATIC_CONTEXTS]; double[] diff = { 0.0, 0.0 }; size_t i; for (i = 0; i < num_contexts; ++i) { size_t curr_histo_ix = self->curr_histogram_ix_ + i; size_t j; entropy[i] = BitsEntropy(histograms[curr_histo_ix].data_, self->alphabet_size_); for (j = 0; j < 2; ++j) { size_t jx = j * num_contexts + i; size_t last_histogram_ix = (j == 0 ? self->last_histogram_ix_0 : self->last_histogram_ix_1) + i; combined_histo[jx] = histograms[curr_histo_ix]; HistogramLiteral.HistogramAddHistogram(&combined_histo[jx], &histograms[last_histogram_ix]); combined_entropy[jx] = BitsEntropy( &combined_histo[jx].data_[0], self->alphabet_size_); diff[j] += combined_entropy[jx] - entropy[i] - last_entropy[jx]; } } if (split->num_types < self->max_block_types_ && diff[0] > self->split_threshold_ && diff[1] > self->split_threshold_) { /* Create new block. */ split->lengths[self->num_blocks_] = (uint)self->block_size_; split->types[self->num_blocks_] = (byte)split->num_types; self->last_histogram_ix_1 = self->last_histogram_ix_0; self->last_histogram_ix_0 = split->num_types * num_contexts; for (i = 0; i < num_contexts; ++i) { last_entropy[num_contexts + i] = last_entropy[i]; last_entropy[i] = entropy[i]; } ++self->num_blocks_; ++split->num_types; self->curr_histogram_ix_ += num_contexts; if (self->curr_histogram_ix_ < *self->histograms_size_) { HistogramLiteral.ClearHistograms( &self->histograms_[self->curr_histogram_ix_], self->num_contexts_); } self->block_size_ = 0; self->merge_last_count_ = 0; self->target_block_size_ = self->min_block_size_; } else if (diff[1] < diff[0] - 20.0) { /* Combine this block with second last block. */ split->lengths[self->num_blocks_] = (uint)self->block_size_; split->types[self->num_blocks_] = split->types[self->num_blocks_ - 2]; size_t tmp = self->last_histogram_ix_0; self->last_histogram_ix_0 = self->last_histogram_ix_1; self->last_histogram_ix_1 = tmp; for (i = 0; i < num_contexts; ++i) { histograms[self->last_histogram_ix_0 + i] = combined_histo[num_contexts + i]; last_entropy[num_contexts + i] = last_entropy[i]; last_entropy[i] = combined_entropy[num_contexts + i]; HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_ + i]); } ++self->num_blocks_; self->block_size_ = 0; self->merge_last_count_ = 0; self->target_block_size_ = self->min_block_size_; } else { /* Combine this block with last block. */ split->lengths[self->num_blocks_ - 1] += (uint)self->block_size_; for (i = 0; i < num_contexts; ++i) { histograms[self->last_histogram_ix_0 + i] = combined_histo[i]; last_entropy[i] = combined_entropy[i]; if (split->num_types == 1) { last_entropy[num_contexts + i] = last_entropy[i]; } HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_ + i]); } self->block_size_ = 0; if (++self->merge_last_count_ > 1) { self->target_block_size_ += self->min_block_size_; } } BrotliFree(ref m, combined_histo); } if (is_final) { *self->histograms_size_ = split->num_types * num_contexts; split->num_blocks = self->num_blocks_; } }
private static unsafe void BrotliSplitBlock(ref MemoryManager m, Command *cmds, size_t num_commands, byte *data, size_t pos, size_t mask, BrotliEncoderParams *params_, BlockSplit *literal_split, BlockSplit *insert_and_copy_split, BlockSplit *dist_split) { { size_t literals_count = CountLiterals(cmds, num_commands); byte * literals = (byte *)BrotliAllocate(ref m, literals_count * sizeof(byte)); /* Create a continuous array of literals. */ CopyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals); /* Create the block split on the array of literals. * Literal histograms have alphabet size 256. */ BlockSplitterLiteral.SplitByteVector( ref m, literals, literals_count, kSymbolsPerLiteralHistogram, kMaxLiteralHistograms, kLiteralStrideLength, kLiteralBlockSwitchCost, params_, literal_split); BrotliFree(ref m, literals); } { /* Compute prefix codes for commands. */ ushort *insert_and_copy_codes = (ushort *)BrotliAllocate(ref m, num_commands * sizeof(ushort)); size_t i; for (i = 0; i < num_commands; ++i) { insert_and_copy_codes[i] = cmds[i].cmd_prefix_; } /* Create the block split on the array of command prefixes. */ BlockSplitterCommand.SplitByteVector( ref m, insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params_, insert_and_copy_split); /* TODO: reuse for distances? */ BrotliFree(ref m, insert_and_copy_codes); } { /* Create a continuous array of distance prefixes. */ ushort *distance_prefixes = (ushort *)BrotliAllocate(ref m, num_commands * sizeof(ushort)); size_t j = 0; size_t i; for (i = 0; i < num_commands; ++i) { Command *cmd = &cmds[i]; if (CommandCopyLen(cmd) != 0 && cmd->cmd_prefix_ >= 128) { distance_prefixes[j++] = cmd->dist_prefix_; } } /* Create the block split on the array of distance prefixes. */ BlockSplitterDistance.SplitByteVector( ref m, distance_prefixes, j, kSymbolsPerDistanceHistogram, kMaxCommandHistograms, kCommandStrideLength, kDistanceBlockSwitchCost, params_, dist_split); BrotliFree(ref m, distance_prefixes); } }
private static unsafe void BrotliDestroyBlockSplit(ref MemoryManager m, BlockSplit *self) { BrotliFree(ref m, self->types); BrotliFree(ref m, self->lengths); }
/* Does either of three things: * (1) emits the current block with a new block type; * (2) emits the current block with the type of the second last block; * (3) merges the current block with the last block. */ public static unsafe void BlockSplitterFinishBlock( BlockSplitterLiteral *self, bool is_final) { BlockSplit * split = self->split_; double * last_entropy = self->last_entropy_; HistogramLiteral *histograms = self->histograms_; self->block_size_ = Math.Max(self->block_size_, self->min_block_size_); if (self->num_blocks_ == 0) { /* Create first block. */ split->lengths[0] = (uint)self->block_size_; split->types[0] = 0; last_entropy[0] = BitsEntropy(histograms[0].data_, self->alphabet_size_); last_entropy[1] = last_entropy[0]; ++self->num_blocks_; ++split->num_types; ++self->curr_histogram_ix_; if (self->curr_histogram_ix_ < *self->histograms_size_) { HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_]); } self->block_size_ = 0; } else if (self->block_size_ > 0) { double entropy = BitsEntropy(histograms[self->curr_histogram_ix_].data_, self->alphabet_size_); HistogramLiteral *combined_histo = stackalloc HistogramLiteral[2]; double * combined_entropy = stackalloc double[2]; double * diff = stackalloc double[2]; size_t j; for (j = 0; j < 2; ++j) { size_t last_histogram_ix = j == 0 ? self->last_histogram_ix_0 : self->last_histogram_ix_1; combined_histo[j] = histograms[self->curr_histogram_ix_]; HistogramLiteral.HistogramAddHistogram(&combined_histo[j], &histograms[last_histogram_ix]); combined_entropy[j] = BitsEntropy( &combined_histo[j].data_[0], self->alphabet_size_); diff[j] = combined_entropy[j] - entropy - last_entropy[j]; } if (split->num_types < BROTLI_MAX_NUMBER_OF_BLOCK_TYPES && diff[0] > self->split_threshold_ && diff[1] > self->split_threshold_) { /* Create new block. */ split->lengths[self->num_blocks_] = (uint)self->block_size_; split->types[self->num_blocks_] = (byte)split->num_types; self->last_histogram_ix_1 = self->last_histogram_ix_0; self->last_histogram_ix_0 = (byte)split->num_types; last_entropy[1] = last_entropy[0]; last_entropy[0] = entropy; ++self->num_blocks_; ++split->num_types; ++self->curr_histogram_ix_; if (self->curr_histogram_ix_ < *self->histograms_size_) { HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_]); } self->block_size_ = 0; self->merge_last_count_ = 0; self->target_block_size_ = self->min_block_size_; } else if (diff[1] < diff[0] - 20.0) { /* Combine this block with second last block. */ split->lengths[self->num_blocks_] = (uint)self->block_size_; split->types[self->num_blocks_] = split->types[self->num_blocks_ - 2]; size_t tmp = self->last_histogram_ix_0; self->last_histogram_ix_0 = self->last_histogram_ix_1; self->last_histogram_ix_1 = tmp; histograms[self->last_histogram_ix_0] = combined_histo[1]; last_entropy[1] = last_entropy[0]; last_entropy[0] = combined_entropy[1]; ++self->num_blocks_; self->block_size_ = 0; HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_]); self->merge_last_count_ = 0; self->target_block_size_ = self->min_block_size_; } else { /* Combine this block with last block. */ split->lengths[self->num_blocks_ - 1] += (uint)self->block_size_; histograms[self->last_histogram_ix_0] = combined_histo[0]; last_entropy[0] = combined_entropy[0]; if (split->num_types == 1) { last_entropy[1] = last_entropy[0]; } self->block_size_ = 0; HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_]); if (++self->merge_last_count_ > 1) { self->target_block_size_ += self->min_block_size_; } } } if (is_final) { *self->histograms_size_ = split->num_types; split->num_blocks = self->num_blocks_; } }