public abstract void CreateBackwardReferences(
ushort *dictionary_hash,
size_t num_bytes, size_t position,
byte *ringbuffer, size_t ringbuffer_mask,
BrotliEncoderParams *params_, HasherHandle hasher, int *dist_cache,
size_t *last_insert_len, Command *commands, size_t *num_commands,
size_t *num_literals);
private static unsafe void BrotliBuildMetaBlockGreedy(ref MemoryManager m,
byte *ringbuffer,
size_t pos,
size_t mask,
byte prev_byte,
byte prev_byte2,
ContextType literal_context_mode,
size_t num_contexts,
uint *static_context_map,
Command *commands,
size_t n_commands,
MetaBlockSplit *mb)
{
if (num_contexts == 1)
{
BrotliBuildMetaBlockGreedyInternal(ref m, ringbuffer, pos, mask, prev_byte,
prev_byte2, literal_context_mode, 1, null, commands, n_commands, mb);
}
else
{
BrotliBuildMetaBlockGreedyInternal(ref m, ringbuffer, pos, mask, prev_byte,
prev_byte2, literal_context_mode, num_contexts, static_context_map,
commands, n_commands, mb);
}
}
private static unsafe void CopyLiteralsToByteArray(Command *cmds,
size_t num_commands,
byte *data,
size_t offset,
size_t mask,
byte *literals)
{
size_t pos = 0;
size_t from_pos = offset & mask;
size_t i;
for (i = 0; i < num_commands; ++i)
{
size_t insert_len = cmds[i].insert_len_;
if (from_pos + insert_len > mask)
{
size_t head_size = mask + 1 - from_pos;
memcpy(literals + pos, data + from_pos, head_size);
from_pos = 0;
pos += head_size;
insert_len -= head_size;
}
if (insert_len > 0)
{
memcpy(literals + pos, data + from_pos, insert_len);
pos += insert_len;
}
from_pos = (from_pos + insert_len + CommandCopyLen(&cmds[i])) & mask;
}
}
private static unsafe void BrotliCreateBackwardReferences(
size_t num_bytes,
size_t position,
byte *ringbuffer,
size_t ringbuffer_mask,
BrotliEncoderParams *params_,
HasherHandle hasher,
int *dist_cache,
size_t *last_insert_len,
Command *commands,
size_t *num_commands,
size_t *num_literals)
{
switch (params_->hasher.type)
{
case 2:
case 3:
case 4:
case 5:
case 6:
case 40:
case 41:
case 42:
case 54:
fixed(ushort *ksdh = kStaticDictionaryHash)
kHashers[params_->hasher.type].CreateBackwardReferences(ksdh, num_bytes,
position, ringbuffer, ringbuffer_mask, params_, hasher, dist_cache,
last_insert_len, commands, num_commands, num_literals);
break;
}
}
private static unsafe void InitInsertCommand(Command *self, size_t insertlen)
{
self->insert_len_ = (uint)insertlen;
self->copy_len_ = 4 << 24;
self->dist_extra_ = 0;
self->dist_prefix_ = BROTLI_NUM_DISTANCE_SHORT_CODES;
GetLengthCode(insertlen, 4, false, &self->cmd_prefix_);
}
private static unsafe void BrotliBuildHistogramsWithContext(
Command *cmds, size_t num_commands,
BlockSplit *literal_split, BlockSplit *insert_and_copy_split,
BlockSplit *dist_split, byte *ringbuffer, size_t start_pos,
size_t mask, byte prev_byte, byte prev_byte2,
ContextType *context_modes, HistogramLiteral *literal_histograms,
HistogramCommand *insert_and_copy_histograms,
HistogramDistance *copy_dist_histograms)
{
size_t pos = start_pos;
BlockSplitIterator literal_it;
BlockSplitIterator insert_and_copy_it;
BlockSplitIterator dist_it;
size_t i;
InitBlockSplitIterator(&literal_it, literal_split);
InitBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split);
InitBlockSplitIterator(&dist_it, dist_split);
for (i = 0; i < num_commands; ++i)
{
Command *cmd = &cmds[i];
size_t j;
BlockSplitIteratorNext(&insert_and_copy_it);
HistogramCommand.HistogramAdd(&insert_and_copy_histograms[insert_and_copy_it.type_],
cmd->cmd_prefix_);
for (j = cmd->insert_len_; j != 0; --j)
{
size_t context;
BlockSplitIteratorNext(&literal_it);
context = context_modes != null ?
((literal_it.type_ << BROTLI_LITERAL_CONTEXT_BITS) +
Context(prev_byte, prev_byte2, context_modes[literal_it.type_])) :
literal_it.type_;
HistogramLiteral.HistogramAdd(&literal_histograms[context],
ringbuffer[pos & mask]);
prev_byte2 = prev_byte;
prev_byte = ringbuffer[pos & mask];
++pos;
}
pos += CommandCopyLen(cmd);
if (CommandCopyLen(cmd) != 0)
{
prev_byte2 = ringbuffer[(pos - 2) & mask];
prev_byte = ringbuffer[(pos - 1) & mask];
if (cmd->cmd_prefix_ >= 128)
{
size_t context;
BlockSplitIteratorNext(&dist_it);
context = (dist_it.type_ << BROTLI_DISTANCE_CONTEXT_BITS) +
CommandDistanceContext(cmd);
HistogramDistance.HistogramAdd(©_dist_histograms[context],
cmd->dist_prefix_);
}
}
}
}
private static unsafe uint CommandDistanceContext(Command *self)
{
uint r = (uint)(self->cmd_prefix_ >> 6);
uint c = (uint)(self->cmd_prefix_ & 7);
if ((r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2))
{
return(c);
}
return(3);
}
private static unsafe size_t CountLiterals(Command *cmds, size_t num_commands)
{
/* Count how many we have. */
size_t total_length = 0;
size_t i;
for (i = 0; i < num_commands; ++i)
{
total_length += cmds[i].insert_len_;
}
return(total_length);
}
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
private static unsafe void BrotliZopfliCreateCommands(size_t num_bytes,
size_t block_start,
size_t max_backward_limit,
ZopfliNode *nodes,
int *dist_cache,
size_t *last_insert_len,
Command *commands,
size_t *num_literals)
{
size_t pos = 0;
uint offset = nodes[0].u.next;
size_t i;
for (i = 0; offset != uint.MaxValue; i++)
{
ZopfliNode *next = &nodes[pos + offset];
size_t copy_length = ZopfliNodeCopyLength(next);
size_t insert_length = next->insert_length;
pos += insert_length;
offset = next->u.next;
if (i == 0)
{
insert_length += *last_insert_len;
*last_insert_len = 0;
}
{
size_t distance = ZopfliNodeCopyDistance(next);
size_t len_code = ZopfliNodeLengthCode(next);
size_t max_distance =
Math.Min(block_start + pos, max_backward_limit);
bool is_dictionary = (distance > max_distance);
size_t dist_code = ZopfliNodeDistanceCode(next);
InitCommand(
&commands[i], insert_length, copy_length, len_code, dist_code);
if (!is_dictionary && dist_code > 0)
{
dist_cache[3] = dist_cache[2];
dist_cache[2] = dist_cache[1];
dist_cache[1] = dist_cache[0];
dist_cache[0] = (int)distance;
}
}
*num_literals += insert_length;
pos += copy_length;
}
*last_insert_len += num_bytes - pos;
}
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
private static unsafe void InitCommand(Command *self, size_t insertlen,
size_t copylen, size_t copylen_code, size_t distance_code)
{
self->insert_len_ = (uint)insertlen;
self->copy_len_ = (uint)(copylen | ((copylen_code ^ copylen) << 24));
/* The distance prefix and extra bits are stored in this Command as if
* npostfix and ndirect were 0, they are only recomputed later after the
* clustering if needed. */
PrefixEncodeCopyDistance(
distance_code, 0, 0, &self->dist_prefix_, &self->dist_extra_);
GetLengthCode(
insertlen, copylen_code, (self->dist_prefix_ == 0),
&self->cmd_prefix_);
}
private static unsafe void BrotliCreateZopfliBackwardReferences(
ref MemoryManager m, size_t num_bytes,
size_t position, byte *ringbuffer, size_t ringbuffer_mask,
BrotliEncoderParams *params_, HasherHandle hasher, int *dist_cache,
size_t *last_insert_len, Command *commands, size_t *num_commands,
size_t *num_literals)
{
size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params_->lgwin);
ZopfliNode *nodes;
nodes = (ZopfliNode *)BrotliAllocate(ref m, (num_bytes + 1) * sizeof(ZopfliNode));
BrotliInitZopfliNodes(nodes, num_bytes + 1);
*num_commands += BrotliZopfliComputeShortestPath(ref m, num_bytes,
position, ringbuffer, ringbuffer_mask, params_, max_backward_limit,
dist_cache, hasher, nodes);
BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit, nodes,
dist_cache, last_insert_len, commands, num_literals);
BrotliFree(ref m, nodes);
}
private static unsafe uint CommandRestoreDistanceCode(Command *self)
{
if (self->dist_prefix_ < BROTLI_NUM_DISTANCE_SHORT_CODES)
{
return(self->dist_prefix_);
}
else
{
uint nbits = self->dist_extra_ >> 24;
uint extra = self->dist_extra_ & 0xffffff;
/* It is assumed that the distance was first encoded with NPOSTFIX = 0 and
* NDIRECT = 0, so the code itself is of this form:
* BROTLI_NUM_DISTANCE_SHORT_CODES + 2 * (nbits - 1) + prefix_bit
* Therefore, the following expression results in (2 + prefix_bit). */
uint prefix =
self->dist_prefix_ + 4u - BROTLI_NUM_DISTANCE_SHORT_CODES - 2u * nbits;
/* Subtract 4 for offset (Chapter 4.) and
* increase by BROTLI_NUM_DISTANCE_SHORT_CODES - 1 */
return((prefix << (int)nbits) + extra + BROTLI_NUM_DISTANCE_SHORT_CODES - 4u);
}
}
public override unsafe void CreateBackwardReferences(
ushort *dictionary_hash,
size_t num_bytes, size_t position,
byte *ringbuffer, size_t ringbuffer_mask,
BrotliEncoderParams *params_, HasherHandle hasher, int *dist_cache,
size_t *last_insert_len, Command *commands, size_t *num_commands,
size_t *num_literals)
{
/* Set maximum distance, see section 9.1. of the spec. */
size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params_->lgwin);
Command *orig_commands = commands;
size_t insert_length = *last_insert_len;
size_t pos_end = position + num_bytes;
size_t store_end = num_bytes >= StoreLookahead()
? position + num_bytes - StoreLookahead() + 1
: position;
/* For speed up heuristics for random data. */
size_t random_heuristics_window_size =
LiteralSpreeLengthForSparseSearch(params_);
size_t apply_random_heuristics = position + random_heuristics_window_size;
/* Minimum score to accept a backward reference. */
score_t kMinScore = BROTLI_SCORE_BASE + 100;
PrepareDistanceCache(hasher, dist_cache);
while (position + HashTypeLength() < pos_end)
{
size_t max_length = pos_end - position;
size_t max_distance = Math.Min(position, max_backward_limit);
HasherSearchResult sr = new HasherSearchResult();
sr.len = 0;
sr.len_x_code = 0;
sr.distance = 0;
sr.score = kMinScore;
if (FindLongestMatch(hasher, dictionary_hash,
ringbuffer, ringbuffer_mask, dist_cache,
position, max_length, max_distance, &sr))
{
/* Found a match. Let's look for something even better ahead. */
int delayed_backward_references_in_row = 0;
--max_length;
for (;; --max_length)
{
score_t cost_diff_lazy = 175;
bool is_match_found;
HasherSearchResult sr2;
sr2.len = params_->quality < MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH
? Math.Min(sr.len - 1, max_length)
: 0;
sr2.len_x_code = 0;
sr2.distance = 0;
sr2.score = kMinScore;
max_distance = Math.Min(position + 1, max_backward_limit);
is_match_found = FindLongestMatch(hasher,
dictionary_hash, ringbuffer, ringbuffer_mask, dist_cache,
position + 1, max_length, max_distance, &sr2);
if (is_match_found && sr2.score >= sr.score + cost_diff_lazy)
{
/* Ok, let's just write one byte for now and start a match from the
* next byte. */
++position;
++insert_length;
sr = sr2;
if (++delayed_backward_references_in_row < 4 &&
position + HashTypeLength() < pos_end)
{
continue;
}
}
break;
}
apply_random_heuristics =
position + 2 * sr.len + random_heuristics_window_size;
max_distance = Math.Min(position, max_backward_limit);
{
/* The first 16 codes are special short-codes,
* and the minimum offset is 1. */
size_t distance_code =
ComputeDistanceCode(sr.distance, max_distance, dist_cache);
if (sr.distance <= max_distance && distance_code > 0)
{
dist_cache[3] = dist_cache[2];
dist_cache[2] = dist_cache[1];
dist_cache[1] = dist_cache[0];
dist_cache[0] = (int)sr.distance;
PrepareDistanceCache(hasher, dist_cache);
}
InitCommand(commands++, insert_length, sr.len, sr.len ^ sr.len_x_code,
distance_code);
}
*num_literals += insert_length;
insert_length = 0;
/* Put the hash keys into the table, if there are enough bytes left.
* Depending on the hasher implementation, it can push all positions
* in the given range or only a subset of them. */
StoreRange(hasher, ringbuffer, ringbuffer_mask, position + 2,
Math.Min(position + sr.len, store_end));
position += sr.len;
}
else
{
++insert_length;
++position;
/* If we have not seen matches for a long time, we can skip some
* match lookups. Unsuccessful match lookups are very very expensive
* and this kind of a heuristic speeds up compression quite
* a lot. */
if (position > apply_random_heuristics)
{
/* Going through uncompressible data, jump. */
if (position >
apply_random_heuristics + 4 * random_heuristics_window_size)
{
/* It is quite a long time since we saw a copy, so we assume
* that this data is not compressible, and store hashes less
* often. Hashes of non compressible data are less likely to
* turn out to be useful in the future, too, so we store less of
* them to not to flood out the hash table of good compressible
* data. */
size_t kMargin =
Math.Max(StoreLookahead() - 1, 4);
size_t pos_jump =
Math.Min(position + 16, pos_end - kMargin);
for (; position < pos_jump; position += 4)
{
Store(hasher, ringbuffer, ringbuffer_mask, position);
insert_length += 4;
}
}
else
{
size_t kMargin =
Math.Max(StoreLookahead() - 1, 2);
size_t pos_jump =
Math.Min(position + 8, pos_end - kMargin);
for (; position < pos_jump; position += 2)
{
Store(hasher, ringbuffer, ringbuffer_mask, position);
insert_length += 2;
}
}
}
}
}
insert_length += pos_end - position;
*last_insert_len = insert_length;
*num_commands += (size_t)(commands - orig_commands);
}
private static unsafe void BrotliBuildMetaBlock(ref MemoryManager m,
byte *ringbuffer,
size_t pos,
size_t mask,
BrotliEncoderParams *params_,
byte prev_byte,
byte prev_byte2,
Command *cmds,
size_t num_commands,
ContextType literal_context_mode,
MetaBlockSplit *mb)
{
/* Histogram ids need to fit in one byte. */
size_t kMaxNumberOfHistograms = 256;
HistogramDistance *distance_histograms;
HistogramLiteral * literal_histograms;
ContextType * literal_context_modes = null;
size_t literal_histograms_size;
size_t distance_histograms_size;
size_t i;
size_t literal_context_multiplier = 1;
BrotliSplitBlock(ref m, cmds, num_commands,
ringbuffer, pos, mask, params_,
&mb->literal_split,
&mb->command_split,
&mb->distance_split);
if (!params_->disable_literal_context_modeling)
{
literal_context_multiplier = 1 << BROTLI_LITERAL_CONTEXT_BITS;
literal_context_modes =
(ContextType *)BrotliAllocate(ref m, mb->literal_split.num_types * sizeof(ContextType));
for (i = 0; i < mb->literal_split.num_types; ++i)
{
literal_context_modes[i] = literal_context_mode;
}
}
literal_histograms_size =
mb->literal_split.num_types * literal_context_multiplier;
literal_histograms =
(HistogramLiteral *)BrotliAllocate(ref m, literal_histograms_size * sizeof(HistogramLiteral));
HistogramLiteral.ClearHistograms(literal_histograms, literal_histograms_size);
distance_histograms_size =
mb->distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS;
distance_histograms =
(HistogramDistance *)BrotliAllocate(ref m, distance_histograms_size * sizeof(HistogramDistance));
HistogramDistance.ClearHistograms(distance_histograms, distance_histograms_size);
mb->command_histograms_size = mb->command_split.num_types;
mb->command_histograms =
(HistogramCommand *)BrotliAllocate(ref m, mb->command_histograms_size * sizeof(HistogramCommand));
HistogramCommand.ClearHistograms(mb->command_histograms, mb->command_histograms_size);
BrotliBuildHistogramsWithContext(cmds, num_commands,
&mb->literal_split, &mb->command_split, &mb->distance_split,
ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes,
literal_histograms, mb->command_histograms, distance_histograms);
BrotliFree(ref m, literal_context_modes);
mb->literal_context_map_size =
mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;
mb->literal_context_map =
(uint *)BrotliAllocate(ref m, mb->literal_context_map_size * sizeof(uint));
mb->literal_histograms_size = mb->literal_context_map_size;
mb->literal_histograms =
(HistogramLiteral *)BrotliAllocate(ref m, mb->literal_histograms_size * sizeof(HistogramLiteral));
ClusterLiteral.BrotliClusterHistograms(ref m, literal_histograms, literal_histograms_size,
kMaxNumberOfHistograms, mb->literal_histograms,
&mb->literal_histograms_size, mb->literal_context_map);
BrotliFree(ref m, literal_histograms);
if (params_->disable_literal_context_modeling)
{
/* Distribute assignment to all contexts. */
for (i = mb->literal_split.num_types; i != 0;)
{
size_t j = 0;
i--;
for (; j < (1 << BROTLI_LITERAL_CONTEXT_BITS); j++)
{
mb->literal_context_map[(i << BROTLI_LITERAL_CONTEXT_BITS) + j] =
mb->literal_context_map[i];
}
}
}
mb->distance_context_map_size =
mb->distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS;
mb->distance_context_map =
(uint *)BrotliAllocate(ref m, mb->distance_context_map_size * sizeof(uint));
mb->distance_histograms_size = mb->distance_context_map_size;
mb->distance_histograms =
(HistogramDistance *)BrotliAllocate(ref m, mb->distance_histograms_size * sizeof(HistogramDistance));
ClusterDistance.BrotliClusterHistograms(ref m, distance_histograms,
mb->distance_context_map_size,
kMaxNumberOfHistograms,
mb->distance_histograms,
&mb->distance_histograms_size,
mb->distance_context_map);
BrotliFree(ref m, distance_histograms);
}
private static unsafe void BrotliSplitBlock(ref MemoryManager m,
Command *cmds,
size_t num_commands,
byte *data,
size_t pos,
size_t mask,
BrotliEncoderParams *params_,
BlockSplit *literal_split,
BlockSplit *insert_and_copy_split,
BlockSplit *dist_split)
{
{
size_t literals_count = CountLiterals(cmds, num_commands);
byte * literals = (byte *)BrotliAllocate(ref m, literals_count * sizeof(byte));
/* Create a continuous array of literals. */
CopyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals);
/* Create the block split on the array of literals.
* Literal histograms have alphabet size 256. */
BlockSplitterLiteral.SplitByteVector(
ref m, literals, literals_count,
kSymbolsPerLiteralHistogram, kMaxLiteralHistograms,
kLiteralStrideLength, kLiteralBlockSwitchCost, params_,
literal_split);
BrotliFree(ref m, literals);
}
{
/* Compute prefix codes for commands. */
ushort *insert_and_copy_codes = (ushort *)BrotliAllocate(ref m, num_commands * sizeof(ushort));
size_t i;
for (i = 0; i < num_commands; ++i)
{
insert_and_copy_codes[i] = cmds[i].cmd_prefix_;
}
/* Create the block split on the array of command prefixes. */
BlockSplitterCommand.SplitByteVector(
ref m, insert_and_copy_codes, num_commands,
kSymbolsPerCommandHistogram, kMaxCommandHistograms,
kCommandStrideLength, kCommandBlockSwitchCost, params_,
insert_and_copy_split);
/* TODO: reuse for distances? */
BrotliFree(ref m, insert_and_copy_codes);
}
{
/* Create a continuous array of distance prefixes. */
ushort *distance_prefixes = (ushort *)BrotliAllocate(ref m, num_commands * sizeof(ushort));
size_t j = 0;
size_t i;
for (i = 0; i < num_commands; ++i)
{
Command *cmd = &cmds[i];
if (CommandCopyLen(cmd) != 0 && cmd->cmd_prefix_ >= 128)
{
distance_prefixes[j++] = cmd->dist_prefix_;
}
}
/* Create the block split on the array of distance prefixes. */
BlockSplitterDistance.SplitByteVector(
ref m, distance_prefixes, j,
kSymbolsPerDistanceHistogram, kMaxCommandHistograms,
kCommandStrideLength, kDistanceBlockSwitchCost, params_,
dist_split);
BrotliFree(ref m, distance_prefixes);
}
}
private static unsafe void BrotliBuildMetaBlockGreedyInternal(
ref MemoryManager m, byte *ringbuffer, size_t pos, size_t mask,
byte prev_byte, byte prev_byte2, ContextType literal_context_mode,
size_t num_contexts, uint *static_context_map,
Command *commands, size_t n_commands, MetaBlockSplit *mb)
{
lit_blocks_union lit_blocks = new lit_blocks_union();
BlockSplitterCommand cmd_blocks;
BlockSplitterDistance dist_blocks;
size_t num_literals = 0;
size_t i;
for (i = 0; i < n_commands; ++i)
{
num_literals += commands[i].insert_len_;
}
if (num_contexts == 1)
{
BlockSplitterLiteral.InitBlockSplitter(ref m, (BlockSplitterLiteral *)&lit_blocks, 256, 512, 400.0,
num_literals, &mb->literal_split, &mb->literal_histograms,
&mb->literal_histograms_size);
}
else
{
InitContextBlockSplitter(ref m, &lit_blocks.ctx, 256, num_contexts, 512, 400.0,
num_literals, &mb->literal_split, &mb->literal_histograms,
&mb->literal_histograms_size);
}
BlockSplitterCommand.InitBlockSplitter(ref m, &cmd_blocks, BROTLI_NUM_COMMAND_SYMBOLS, 1024,
500.0, n_commands, &mb->command_split, &mb->command_histograms,
&mb->command_histograms_size);
BlockSplitterDistance.InitBlockSplitter(ref m, &dist_blocks, 64, 512, 100.0, n_commands,
&mb->distance_split, &mb->distance_histograms,
&mb->distance_histograms_size);
for (i = 0; i < n_commands; ++i)
{
Command cmd = commands[i];
size_t j;
BlockSplitterCommand.BlockSplitterAddSymbol(&cmd_blocks, cmd.cmd_prefix_);
for (j = cmd.insert_len_; j != 0; --j)
{
byte literal = ringbuffer[pos & mask];
if (num_contexts == 1)
{
BlockSplitterLiteral.BlockSplitterAddSymbol((BlockSplitterLiteral *)&lit_blocks, literal);
}
else
{
size_t context = Context(prev_byte, prev_byte2, literal_context_mode);
ContextBlockSplitterAddSymbol(&lit_blocks.ctx, ref m, literal,
static_context_map[context]);
}
prev_byte2 = prev_byte;
prev_byte = literal;
++pos;
}
pos += CommandCopyLen(&cmd);
if (CommandCopyLen(&cmd) != 0)
{
prev_byte2 = ringbuffer[(pos - 2) & mask];
prev_byte = ringbuffer[(pos - 1) & mask];
if (cmd.cmd_prefix_ >= 128)
{
BlockSplitterDistance.BlockSplitterAddSymbol(&dist_blocks, cmd.dist_prefix_);
}
}
}
if (num_contexts == 1)
{
BlockSplitterLiteral.BlockSplitterFinishBlock(
(BlockSplitterLiteral *)&lit_blocks, /* is_final = */ true);
}
else
{
ContextBlockSplitterFinishBlock(
&lit_blocks.ctx, ref m, /* is_final = */ true);
}
BlockSplitterCommand.BlockSplitterFinishBlock(&cmd_blocks, /* is_final = */ true);
BlockSplitterDistance.BlockSplitterFinishBlock(&dist_blocks, /* is_final = */ true);
if (num_contexts > 1)
{
MapStaticContexts(ref m, num_contexts, static_context_map, mb);
}
}
private static unsafe uint CommandCopyLen(Command *self)
{
return(self->copy_len_ & 0xFFFFFF);
}
private static unsafe uint CommandCopyLenCode(Command *self)
{
return((self->copy_len_ & 0xFFFFFF) ^ (self->copy_len_ >> 24));
}
public override unsafe void CreateBackwardReferences(ushort *dictionary_hash, size_t num_bytes,
size_t position, byte *ringbuffer, size_t ringbuffer_mask, BrotliEncoderParams *params_,
HasherHandle hasher, int *dist_cache, size_t *last_insert_len, Command *commands, size_t *num_commands,
size_t *num_literals)
{
throw new InvalidOperationException();
}
private static unsafe void BrotliCreateHqZopfliBackwardReferences(
ref MemoryManager m, size_t num_bytes,
size_t position, byte *ringbuffer, size_t ringbuffer_mask,
BrotliEncoderParams *params_, HasherHandle hasher, int *dist_cache,
size_t *last_insert_len, Command *commands, size_t *num_commands,
size_t *num_literals)
{
size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params_->lgwin);
uint * num_matches = (uint *)BrotliAllocate(ref m, num_bytes * sizeof(uint));
size_t matches_size = 4 * num_bytes;
Hasher h = kHashers[10];
size_t store_end = num_bytes >= h.StoreLookahead()
? position + num_bytes - h.StoreLookahead() + 1
: position;
size_t cur_match_pos = 0;
size_t i;
size_t orig_num_literals;
size_t orig_last_insert_len;
int * orig_dist_cache = stackalloc int[4];
size_t orig_num_commands;
ZopfliCostModel model;
ZopfliNode * nodes;
BackwardMatch * matches = (BackwardMatch *)BrotliAllocate(ref m, matches_size * sizeof(BackwardMatch));
for (i = 0; i + h.HashTypeLength() - 1 < num_bytes; ++i)
{
size_t pos = position + i;
size_t max_distance = Math.Min(pos, max_backward_limit);
size_t max_length = num_bytes - i;
size_t num_found_matches;
size_t cur_match_end;
/* Ensure that we have enough free slots. */
BrotliEnsureCapacity(ref m, sizeof(BackwardMatch), (void **)&matches, &matches_size, cur_match_pos + MAX_NUM_MATCHES_H10);
num_found_matches = HashToBinaryTreeH10.FindAllMatches(hasher, ringbuffer,
ringbuffer_mask, pos, max_length, max_distance, params_,
&matches[cur_match_pos]);
cur_match_end = cur_match_pos + num_found_matches;
num_matches[i] = (uint)num_found_matches;
if (num_found_matches > 0)
{
size_t match_len = BackwardMatchLength(&matches[cur_match_end - 1]);
if (match_len > MAX_ZOPFLI_LEN_QUALITY_11)
{
size_t skip = match_len - 1;
matches[cur_match_pos++] = matches[cur_match_end - 1];
num_matches[i] = 1;
/* Add the tail of the copy to the hasher. */
h.StoreRange(hasher, ringbuffer, ringbuffer_mask, pos + 1,
Math.Min(pos + match_len, store_end));
memset(&num_matches[i + 1], 0, skip * sizeof(uint));
i += skip;
}
else
{
cur_match_pos = cur_match_end;
}
}
}
orig_num_literals = *num_literals;
orig_last_insert_len = *last_insert_len;
memcpy(orig_dist_cache, dist_cache, 4 * sizeof(int));
orig_num_commands = *num_commands;
nodes = (ZopfliNode *)BrotliAllocate(ref m, (num_bytes + 1) * sizeof(ZopfliNode));
InitZopfliCostModel(ref m, &model, num_bytes);
for (i = 0; i < 2; i++)
{
BrotliInitZopfliNodes(nodes, num_bytes + 1);
if (i == 0)
{
ZopfliCostModelSetFromLiteralCosts(
&model, position, ringbuffer, ringbuffer_mask);
}
else
{
ZopfliCostModelSetFromCommands(&model, position, ringbuffer,
ringbuffer_mask, commands, *num_commands - orig_num_commands,
orig_last_insert_len);
}
*num_commands = orig_num_commands;
*num_literals = orig_num_literals;
*last_insert_len = orig_last_insert_len;
memcpy(dist_cache, orig_dist_cache, 4 * sizeof(int));
*num_commands += ZopfliIterate(num_bytes, position, ringbuffer,
ringbuffer_mask, params_, max_backward_limit, dist_cache,
&model, num_matches, matches, nodes);
BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit,
nodes, dist_cache, last_insert_len, commands, num_literals);
}
CleanupZopfliCostModel(ref m, &model);
BrotliFree(ref m, nodes);
BrotliFree(ref m, matches);
BrotliFree(ref m, num_matches);
}
private static unsafe void ZopfliCostModelSetFromCommands(ZopfliCostModel *self,
size_t position,
byte *ringbuffer,
size_t ringbuffer_mask,
Command *commands,
size_t num_commands,
size_t last_insert_len)
{
uint * histogram_literal = stackalloc uint[BROTLI_NUM_LITERAL_SYMBOLS];
uint * histogram_cmd = stackalloc uint[BROTLI_NUM_COMMAND_SYMBOLS];
uint * histogram_dist = stackalloc uint[BROTLI_NUM_DISTANCE_SYMBOLS];
float *cost_literal = stackalloc float[BROTLI_NUM_LITERAL_SYMBOLS];
size_t pos = position - last_insert_len;
float min_cost_cmd = kInfinity;
size_t i;
float *cost_cmd = self->cost_cmd_;
memset(histogram_literal, 0, BROTLI_NUM_LITERAL_SYMBOLS * sizeof(uint));
memset(histogram_cmd, 0, BROTLI_NUM_COMMAND_SYMBOLS * sizeof(uint));
memset(histogram_dist, 0, BROTLI_NUM_DISTANCE_SYMBOLS * sizeof(uint));
for (i = 0; i < num_commands; i++)
{
size_t inslength = commands[i].insert_len_;
size_t copylength = CommandCopyLen(&commands[i]);
size_t distcode = commands[i].dist_prefix_;
size_t cmdcode = commands[i].cmd_prefix_;
size_t j;
histogram_cmd[cmdcode]++;
if (cmdcode >= 128)
{
histogram_dist[distcode]++;
}
for (j = 0; j < inslength; j++)
{
histogram_literal[ringbuffer[(pos + j) & ringbuffer_mask]]++;
}
pos += inslength + copylength;
}
SetCost(histogram_literal, BROTLI_NUM_LITERAL_SYMBOLS, cost_literal);
SetCost(histogram_cmd, BROTLI_NUM_COMMAND_SYMBOLS, cost_cmd);
SetCost(histogram_dist, BROTLI_NUM_DISTANCE_SYMBOLS, self->cost_dist_);
for (i = 0; i < BROTLI_NUM_COMMAND_SYMBOLS; ++i)
{
min_cost_cmd = Math.Min(min_cost_cmd, cost_cmd[i]);
}
self->min_cost_cmd_ = min_cost_cmd;
{
float *literal_costs = self->literal_costs_;
size_t num_bytes = self->num_bytes_;
literal_costs[0] = 0.0f;
for (i = 0; i < num_bytes; ++i)
{
literal_costs[i + 1] = literal_costs[i] +
cost_literal[ringbuffer[(position + i) & ringbuffer_mask]];
}
}
}
