public static unsafe void InitBlockSplitter(
ref MemoryManager m, BlockSplitterDistance *self, size_t alphabet_size,
size_t min_block_size, double split_threshold, size_t num_symbols,
BlockSplit *split, HistogramDistance **histograms, size_t *histograms_size)
{
size_t max_num_blocks = num_symbols / min_block_size + 1;
/* We have to allocate one more histogram than the maximum number of block
* types for the current histogram when the meta-block is too big. */
size_t max_num_types =
Math.Min(max_num_blocks, BROTLI_MAX_NUMBER_OF_BLOCK_TYPES + 1);
self->alphabet_size_ = alphabet_size;
self->min_block_size_ = min_block_size;
self->split_threshold_ = split_threshold;
self->num_blocks_ = 0;
self->split_ = split;
self->histograms_size_ = histograms_size;
self->target_block_size_ = min_block_size;
self->block_size_ = 0;
self->curr_histogram_ix_ = 0;
self->merge_last_count_ = 0;
BrotliEnsureCapacity(ref m, sizeof(byte), (void **)&split->types, &split->types_alloc_size, max_num_blocks);
BrotliEnsureCapacity(ref m, sizeof(uint), (void **)&split->lengths, &split->lengths_alloc_size, max_num_blocks);
self->split_->num_blocks = max_num_blocks;
*histograms_size = max_num_types;
*histograms = (HistogramDistance *)BrotliAllocate(ref m, *histograms_size * sizeof(HistogramDistance));
self->histograms_ = *histograms;
/* Clear only current histogram. */
HistogramDistance.HistogramClear(&self->histograms_[0]);
self->last_histogram_ix_0 = self->last_histogram_ix_1 = 0;
}
private static unsafe void InitBlockSplitIterator(BlockSplitIterator *self,
BlockSplit *split)
{
self->split_ = split;
self->idx_ = 0;
self->type_ = 0;
self->length_ = split->lengths != null ? split->lengths[0] : 0;
}
private static unsafe void BrotliInitBlockSplit(BlockSplit *self)
{
self->num_types = 0;
self->num_blocks = 0;
self->types = null;
self->lengths = null;
self->types_alloc_size = 0;
self->lengths_alloc_size = 0;
}
private static unsafe void BrotliBuildHistogramsWithContext(
Command *cmds, size_t num_commands,
BlockSplit *literal_split, BlockSplit *insert_and_copy_split,
BlockSplit *dist_split, byte *ringbuffer, size_t start_pos,
size_t mask, byte prev_byte, byte prev_byte2,
ContextType *context_modes, HistogramLiteral *literal_histograms,
HistogramCommand *insert_and_copy_histograms,
HistogramDistance *copy_dist_histograms)
{
size_t pos = start_pos;
BlockSplitIterator literal_it;
BlockSplitIterator insert_and_copy_it;
BlockSplitIterator dist_it;
size_t i;
InitBlockSplitIterator(&literal_it, literal_split);
InitBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split);
InitBlockSplitIterator(&dist_it, dist_split);
for (i = 0; i < num_commands; ++i)
{
Command *cmd = &cmds[i];
size_t j;
BlockSplitIteratorNext(&insert_and_copy_it);
HistogramCommand.HistogramAdd(&insert_and_copy_histograms[insert_and_copy_it.type_],
cmd->cmd_prefix_);
for (j = cmd->insert_len_; j != 0; --j)
{
size_t context;
BlockSplitIteratorNext(&literal_it);
context = context_modes != null ?
((literal_it.type_ << BROTLI_LITERAL_CONTEXT_BITS) +
Context(prev_byte, prev_byte2, context_modes[literal_it.type_])) :
literal_it.type_;
HistogramLiteral.HistogramAdd(&literal_histograms[context],
ringbuffer[pos & mask]);
prev_byte2 = prev_byte;
prev_byte = ringbuffer[pos & mask];
++pos;
}
pos += CommandCopyLen(cmd);
if (CommandCopyLen(cmd) != 0)
{
prev_byte2 = ringbuffer[(pos - 2) & mask];
prev_byte = ringbuffer[(pos - 1) & mask];
if (cmd->cmd_prefix_ >= 128)
{
size_t context;
BlockSplitIteratorNext(&dist_it);
context = (dist_it.type_ << BROTLI_DISTANCE_CONTEXT_BITS) +
CommandDistanceContext(cmd);
HistogramDistance.HistogramAdd(©_dist_histograms[context],
cmd->dist_prefix_);
}
}
}
}
/* Does either of three things:
* (1) emits the current block with a new block type;
* (2) emits the current block with the type of the second last block;
* (3) merges the current block with the last block. */
private static unsafe void ContextBlockSplitterFinishBlock(
ContextBlockSplitter *self, ref MemoryManager m, bool is_final)
{
BlockSplit * split = self->split_;
size_t num_contexts = self->num_contexts_;
double * last_entropy = self->last_entropy_;
HistogramLiteral *histograms = self->histograms_;
if (self->block_size_ < self->min_block_size_)
{
self->block_size_ = self->min_block_size_;
}
if (self->num_blocks_ == 0)
{
size_t i;
/* Create first block. */
split->lengths[0] = (uint)self->block_size_;
split->types[0] = 0;
for (i = 0; i < num_contexts; ++i)
{
last_entropy[i] =
BitsEntropy(histograms[i].data_, self->alphabet_size_);
last_entropy[num_contexts + i] = last_entropy[i];
}
++self->num_blocks_;
++split->num_types;
self->curr_histogram_ix_ += num_contexts;
if (self->curr_histogram_ix_ < *self->histograms_size_)
{
HistogramLiteral.ClearHistograms(
&self->histograms_[self->curr_histogram_ix_], self->num_contexts_);
}
self->block_size_ = 0;
}
else if (self->block_size_ > 0)
{
/* Try merging the set of histograms for the current block type with the
* respective set of histograms for the last and second last block types.
* Decide over the split based on the total reduction of entropy across
* all contexts. */
double * entropy = stackalloc double[BROTLI_MAX_STATIC_CONTEXTS];
HistogramLiteral *combined_histo =
(HistogramLiteral *)BrotliAllocate(ref m, 2 * num_contexts * sizeof(HistogramLiteral));
double[] combined_entropy = new double[2 * BROTLI_MAX_STATIC_CONTEXTS];
double[] diff = { 0.0, 0.0 };
size_t i;
for (i = 0; i < num_contexts; ++i)
{
size_t curr_histo_ix = self->curr_histogram_ix_ + i;
size_t j;
entropy[i] = BitsEntropy(histograms[curr_histo_ix].data_,
self->alphabet_size_);
for (j = 0; j < 2; ++j)
{
size_t jx = j * num_contexts + i;
size_t last_histogram_ix = (j == 0 ? self->last_histogram_ix_0 : self->last_histogram_ix_1) + i;
combined_histo[jx] = histograms[curr_histo_ix];
HistogramLiteral.HistogramAddHistogram(&combined_histo[jx],
&histograms[last_histogram_ix]);
combined_entropy[jx] = BitsEntropy(
&combined_histo[jx].data_[0], self->alphabet_size_);
diff[j] += combined_entropy[jx] - entropy[i] - last_entropy[jx];
}
}
if (split->num_types < self->max_block_types_ &&
diff[0] > self->split_threshold_ &&
diff[1] > self->split_threshold_)
{
/* Create new block. */
split->lengths[self->num_blocks_] = (uint)self->block_size_;
split->types[self->num_blocks_] = (byte)split->num_types;
self->last_histogram_ix_1 = self->last_histogram_ix_0;
self->last_histogram_ix_0 = split->num_types * num_contexts;
for (i = 0; i < num_contexts; ++i)
{
last_entropy[num_contexts + i] = last_entropy[i];
last_entropy[i] = entropy[i];
}
++self->num_blocks_;
++split->num_types;
self->curr_histogram_ix_ += num_contexts;
if (self->curr_histogram_ix_ < *self->histograms_size_)
{
HistogramLiteral.ClearHistograms(
&self->histograms_[self->curr_histogram_ix_], self->num_contexts_);
}
self->block_size_ = 0;
self->merge_last_count_ = 0;
self->target_block_size_ = self->min_block_size_;
}
else if (diff[1] < diff[0] - 20.0)
{
/* Combine this block with second last block. */
split->lengths[self->num_blocks_] = (uint)self->block_size_;
split->types[self->num_blocks_] = split->types[self->num_blocks_ - 2];
size_t tmp = self->last_histogram_ix_0;
self->last_histogram_ix_0 = self->last_histogram_ix_1;
self->last_histogram_ix_1 = tmp;
for (i = 0; i < num_contexts; ++i)
{
histograms[self->last_histogram_ix_0 + i] =
combined_histo[num_contexts + i];
last_entropy[num_contexts + i] = last_entropy[i];
last_entropy[i] = combined_entropy[num_contexts + i];
HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_ + i]);
}
++self->num_blocks_;
self->block_size_ = 0;
self->merge_last_count_ = 0;
self->target_block_size_ = self->min_block_size_;
}
else
{
/* Combine this block with last block. */
split->lengths[self->num_blocks_ - 1] += (uint)self->block_size_;
for (i = 0; i < num_contexts; ++i)
{
histograms[self->last_histogram_ix_0 + i] = combined_histo[i];
last_entropy[i] = combined_entropy[i];
if (split->num_types == 1)
{
last_entropy[num_contexts + i] = last_entropy[i];
}
HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_ + i]);
}
self->block_size_ = 0;
if (++self->merge_last_count_ > 1)
{
self->target_block_size_ += self->min_block_size_;
}
}
BrotliFree(ref m, combined_histo);
}
if (is_final)
{
*self->histograms_size_ = split->num_types * num_contexts;
split->num_blocks = self->num_blocks_;
}
}
private static unsafe void BrotliSplitBlock(ref MemoryManager m,
Command *cmds,
size_t num_commands,
byte *data,
size_t pos,
size_t mask,
BrotliEncoderParams *params_,
BlockSplit *literal_split,
BlockSplit *insert_and_copy_split,
BlockSplit *dist_split)
{
{
size_t literals_count = CountLiterals(cmds, num_commands);
byte * literals = (byte *)BrotliAllocate(ref m, literals_count * sizeof(byte));
/* Create a continuous array of literals. */
CopyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals);
/* Create the block split on the array of literals.
* Literal histograms have alphabet size 256. */
BlockSplitterLiteral.SplitByteVector(
ref m, literals, literals_count,
kSymbolsPerLiteralHistogram, kMaxLiteralHistograms,
kLiteralStrideLength, kLiteralBlockSwitchCost, params_,
literal_split);
BrotliFree(ref m, literals);
}
{
/* Compute prefix codes for commands. */
ushort *insert_and_copy_codes = (ushort *)BrotliAllocate(ref m, num_commands * sizeof(ushort));
size_t i;
for (i = 0; i < num_commands; ++i)
{
insert_and_copy_codes[i] = cmds[i].cmd_prefix_;
}
/* Create the block split on the array of command prefixes. */
BlockSplitterCommand.SplitByteVector(
ref m, insert_and_copy_codes, num_commands,
kSymbolsPerCommandHistogram, kMaxCommandHistograms,
kCommandStrideLength, kCommandBlockSwitchCost, params_,
insert_and_copy_split);
/* TODO: reuse for distances? */
BrotliFree(ref m, insert_and_copy_codes);
}
{
/* Create a continuous array of distance prefixes. */
ushort *distance_prefixes = (ushort *)BrotliAllocate(ref m, num_commands * sizeof(ushort));
size_t j = 0;
size_t i;
for (i = 0; i < num_commands; ++i)
{
Command *cmd = &cmds[i];
if (CommandCopyLen(cmd) != 0 && cmd->cmd_prefix_ >= 128)
{
distance_prefixes[j++] = cmd->dist_prefix_;
}
}
/* Create the block split on the array of distance prefixes. */
BlockSplitterDistance.SplitByteVector(
ref m, distance_prefixes, j,
kSymbolsPerDistanceHistogram, kMaxCommandHistograms,
kCommandStrideLength, kDistanceBlockSwitchCost, params_,
dist_split);
BrotliFree(ref m, distance_prefixes);
}
}
private static unsafe void BrotliDestroyBlockSplit(ref MemoryManager m, BlockSplit *self)
{
BrotliFree(ref m, self->types);
BrotliFree(ref m, self->lengths);
}
/* Does either of three things:
* (1) emits the current block with a new block type;
* (2) emits the current block with the type of the second last block;
* (3) merges the current block with the last block. */
public static unsafe void BlockSplitterFinishBlock(
BlockSplitterLiteral *self, bool is_final)
{
BlockSplit * split = self->split_;
double * last_entropy = self->last_entropy_;
HistogramLiteral *histograms = self->histograms_;
self->block_size_ =
Math.Max(self->block_size_, self->min_block_size_);
if (self->num_blocks_ == 0)
{
/* Create first block. */
split->lengths[0] = (uint)self->block_size_;
split->types[0] = 0;
last_entropy[0] =
BitsEntropy(histograms[0].data_, self->alphabet_size_);
last_entropy[1] = last_entropy[0];
++self->num_blocks_;
++split->num_types;
++self->curr_histogram_ix_;
if (self->curr_histogram_ix_ < *self->histograms_size_)
{
HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_]);
}
self->block_size_ = 0;
}
else if (self->block_size_ > 0)
{
double entropy = BitsEntropy(histograms[self->curr_histogram_ix_].data_,
self->alphabet_size_);
HistogramLiteral *combined_histo = stackalloc HistogramLiteral[2];
double * combined_entropy = stackalloc double[2];
double * diff = stackalloc double[2];
size_t j;
for (j = 0; j < 2; ++j)
{
size_t last_histogram_ix = j == 0 ? self->last_histogram_ix_0 : self->last_histogram_ix_1;
combined_histo[j] = histograms[self->curr_histogram_ix_];
HistogramLiteral.HistogramAddHistogram(&combined_histo[j],
&histograms[last_histogram_ix]);
combined_entropy[j] = BitsEntropy(
&combined_histo[j].data_[0], self->alphabet_size_);
diff[j] = combined_entropy[j] - entropy - last_entropy[j];
}
if (split->num_types < BROTLI_MAX_NUMBER_OF_BLOCK_TYPES &&
diff[0] > self->split_threshold_ &&
diff[1] > self->split_threshold_)
{
/* Create new block. */
split->lengths[self->num_blocks_] = (uint)self->block_size_;
split->types[self->num_blocks_] = (byte)split->num_types;
self->last_histogram_ix_1 = self->last_histogram_ix_0;
self->last_histogram_ix_0 = (byte)split->num_types;
last_entropy[1] = last_entropy[0];
last_entropy[0] = entropy;
++self->num_blocks_;
++split->num_types;
++self->curr_histogram_ix_;
if (self->curr_histogram_ix_ < *self->histograms_size_)
{
HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_]);
}
self->block_size_ = 0;
self->merge_last_count_ = 0;
self->target_block_size_ = self->min_block_size_;
}
else if (diff[1] < diff[0] - 20.0)
{
/* Combine this block with second last block. */
split->lengths[self->num_blocks_] = (uint)self->block_size_;
split->types[self->num_blocks_] = split->types[self->num_blocks_ - 2];
size_t tmp = self->last_histogram_ix_0;
self->last_histogram_ix_0 = self->last_histogram_ix_1;
self->last_histogram_ix_1 = tmp;
histograms[self->last_histogram_ix_0] = combined_histo[1];
last_entropy[1] = last_entropy[0];
last_entropy[0] = combined_entropy[1];
++self->num_blocks_;
self->block_size_ = 0;
HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_]);
self->merge_last_count_ = 0;
self->target_block_size_ = self->min_block_size_;
}
else
{
/* Combine this block with last block. */
split->lengths[self->num_blocks_ - 1] += (uint)self->block_size_;
histograms[self->last_histogram_ix_0] = combined_histo[0];
last_entropy[0] = combined_entropy[0];
if (split->num_types == 1)
{
last_entropy[1] = last_entropy[0];
}
self->block_size_ = 0;
HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_]);
if (++self->merge_last_count_ > 1)
{
self->target_block_size_ += self->min_block_size_;
}
}
}
if (is_final)
{
*self->histograms_size_ = split->num_types;
split->num_blocks = self->num_blocks_;
}
}