/* Adds the next symbol to the current block type and context. When the
* current block reaches the target size, decides on merging the block. */
private static unsafe void ContextBlockSplitterAddSymbol(
ContextBlockSplitter *self, ref MemoryManager m,
size_t symbol, size_t context)
{
HistogramLiteral.HistogramAdd(&self->histograms_[self->curr_histogram_ix_ + context],
symbol);
++self->block_size_;
if (self->block_size_ == self->target_block_size_)
{
ContextBlockSplitterFinishBlock(self, ref m, /* is_final = */ false);
}
}
private static unsafe void InitContextBlockSplitter(
ref MemoryManager m, ContextBlockSplitter *self, size_t alphabet_size,
size_t num_contexts, size_t min_block_size, double split_threshold,
size_t num_symbols, BlockSplit *split, HistogramLiteral **histograms,
size_t *histograms_size)
{
size_t max_num_blocks = num_symbols / min_block_size + 1;
size_t max_num_types;
self->alphabet_size_ = alphabet_size;
self->num_contexts_ = num_contexts;
self->max_block_types_ = BROTLI_MAX_NUMBER_OF_BLOCK_TYPES / num_contexts;
self->min_block_size_ = min_block_size;
self->split_threshold_ = split_threshold;
self->num_blocks_ = 0;
self->split_ = split;
self->histograms_size_ = histograms_size;
self->target_block_size_ = min_block_size;
self->block_size_ = 0;
self->curr_histogram_ix_ = 0;
self->merge_last_count_ = 0;
/* We have to allocate one more histogram than the maximum number of block
* types for the current histogram when the meta-block is too big. */
max_num_types =
Math.Min(max_num_blocks, self->max_block_types_ + 1);
BrotliEnsureCapacity(ref m, sizeof(byte), (void **)&split->types, &split->types_alloc_size, max_num_blocks);
BrotliEnsureCapacity(ref m, sizeof(uint), (void **)&split->lengths, &split->lengths_alloc_size, max_num_blocks);
split->num_blocks = max_num_blocks;
*histograms_size = max_num_types * num_contexts;
*histograms = (HistogramLiteral *)BrotliAllocate(ref m, *histograms_size * sizeof(HistogramLiteral));
self->histograms_ = *histograms;
/* Clear only current histogram. */
HistogramLiteral.ClearHistograms(&self->histograms_[0], num_contexts);
self->last_histogram_ix_0 = self->last_histogram_ix_1 = 0;
}
/* Does either of three things:
* (1) emits the current block with a new block type;
* (2) emits the current block with the type of the second last block;
* (3) merges the current block with the last block. */
private static unsafe void ContextBlockSplitterFinishBlock(
ContextBlockSplitter *self, ref MemoryManager m, bool is_final)
{
BlockSplit * split = self->split_;
size_t num_contexts = self->num_contexts_;
double * last_entropy = self->last_entropy_;
HistogramLiteral *histograms = self->histograms_;
if (self->block_size_ < self->min_block_size_)
{
self->block_size_ = self->min_block_size_;
}
if (self->num_blocks_ == 0)
{
size_t i;
/* Create first block. */
split->lengths[0] = (uint)self->block_size_;
split->types[0] = 0;
for (i = 0; i < num_contexts; ++i)
{
last_entropy[i] =
BitsEntropy(histograms[i].data_, self->alphabet_size_);
last_entropy[num_contexts + i] = last_entropy[i];
}
++self->num_blocks_;
++split->num_types;
self->curr_histogram_ix_ += num_contexts;
if (self->curr_histogram_ix_ < *self->histograms_size_)
{
HistogramLiteral.ClearHistograms(
&self->histograms_[self->curr_histogram_ix_], self->num_contexts_);
}
self->block_size_ = 0;
}
else if (self->block_size_ > 0)
{
/* Try merging the set of histograms for the current block type with the
* respective set of histograms for the last and second last block types.
* Decide over the split based on the total reduction of entropy across
* all contexts. */
double * entropy = stackalloc double[BROTLI_MAX_STATIC_CONTEXTS];
HistogramLiteral *combined_histo =
(HistogramLiteral *)BrotliAllocate(ref m, 2 * num_contexts * sizeof(HistogramLiteral));
double[] combined_entropy = new double[2 * BROTLI_MAX_STATIC_CONTEXTS];
double[] diff = { 0.0, 0.0 };
size_t i;
for (i = 0; i < num_contexts; ++i)
{
size_t curr_histo_ix = self->curr_histogram_ix_ + i;
size_t j;
entropy[i] = BitsEntropy(histograms[curr_histo_ix].data_,
self->alphabet_size_);
for (j = 0; j < 2; ++j)
{
size_t jx = j * num_contexts + i;
size_t last_histogram_ix = (j == 0 ? self->last_histogram_ix_0 : self->last_histogram_ix_1) + i;
combined_histo[jx] = histograms[curr_histo_ix];
HistogramLiteral.HistogramAddHistogram(&combined_histo[jx],
&histograms[last_histogram_ix]);
combined_entropy[jx] = BitsEntropy(
&combined_histo[jx].data_[0], self->alphabet_size_);
diff[j] += combined_entropy[jx] - entropy[i] - last_entropy[jx];
}
}
if (split->num_types < self->max_block_types_ &&
diff[0] > self->split_threshold_ &&
diff[1] > self->split_threshold_)
{
/* Create new block. */
split->lengths[self->num_blocks_] = (uint)self->block_size_;
split->types[self->num_blocks_] = (byte)split->num_types;
self->last_histogram_ix_1 = self->last_histogram_ix_0;
self->last_histogram_ix_0 = split->num_types * num_contexts;
for (i = 0; i < num_contexts; ++i)
{
last_entropy[num_contexts + i] = last_entropy[i];
last_entropy[i] = entropy[i];
}
++self->num_blocks_;
++split->num_types;
self->curr_histogram_ix_ += num_contexts;
if (self->curr_histogram_ix_ < *self->histograms_size_)
{
HistogramLiteral.ClearHistograms(
&self->histograms_[self->curr_histogram_ix_], self->num_contexts_);
}
self->block_size_ = 0;
self->merge_last_count_ = 0;
self->target_block_size_ = self->min_block_size_;
}
else if (diff[1] < diff[0] - 20.0)
{
/* Combine this block with second last block. */
split->lengths[self->num_blocks_] = (uint)self->block_size_;
split->types[self->num_blocks_] = split->types[self->num_blocks_ - 2];
size_t tmp = self->last_histogram_ix_0;
self->last_histogram_ix_0 = self->last_histogram_ix_1;
self->last_histogram_ix_1 = tmp;
for (i = 0; i < num_contexts; ++i)
{
histograms[self->last_histogram_ix_0 + i] =
combined_histo[num_contexts + i];
last_entropy[num_contexts + i] = last_entropy[i];
last_entropy[i] = combined_entropy[num_contexts + i];
HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_ + i]);
}
++self->num_blocks_;
self->block_size_ = 0;
self->merge_last_count_ = 0;
self->target_block_size_ = self->min_block_size_;
}
else
{
/* Combine this block with last block. */
split->lengths[self->num_blocks_ - 1] += (uint)self->block_size_;
for (i = 0; i < num_contexts; ++i)
{
histograms[self->last_histogram_ix_0 + i] = combined_histo[i];
last_entropy[i] = combined_entropy[i];
if (split->num_types == 1)
{
last_entropy[num_contexts + i] = last_entropy[i];
}
HistogramLiteral.HistogramClear(&histograms[self->curr_histogram_ix_ + i]);
}
self->block_size_ = 0;
if (++self->merge_last_count_ > 1)
{
self->target_block_size_ += self->min_block_size_;
}
}
BrotliFree(ref m, combined_histo);
}
if (is_final)
{
*self->histograms_size_ = split->num_types * num_contexts;
split->num_blocks = self->num_blocks_;
}
}
