private static unsafe void ZopfliCostModelSetFromLiteralCosts(ZopfliCostModel *self,
size_t position,
byte *ringbuffer,
size_t ringbuffer_mask)
{
float *literal_costs = self->literal_costs_;
float *cost_dist = self->cost_dist_;
float *cost_cmd = self->cost_cmd_;
size_t num_bytes = self->num_bytes_;
size_t i;
BrotliEstimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask,
ringbuffer, &literal_costs[1]);
literal_costs[0] = 0.0f;
for (i = 0; i < num_bytes; ++i)
{
literal_costs[i + 1] += literal_costs[i];
}
for (i = 0; i < BROTLI_NUM_COMMAND_SYMBOLS; ++i)
{
cost_cmd[i] = (float)FastLog2(11 + (uint)i);
}
for (i = 0; i < BROTLI_NUM_DISTANCE_SYMBOLS; ++i)
{
cost_dist[i] = (float)FastLog2(20 + (uint)i);
}
self->min_cost_cmd_ = (float)FastLog2(11);
}
private static unsafe size_t ZopfliIterate(size_t num_bytes,
size_t position,
byte *ringbuffer,
size_t ringbuffer_mask,
BrotliEncoderParams *params_,
size_t max_backward_limit,
int *dist_cache,
ZopfliCostModel *model,
uint *num_matches,
BackwardMatch *matches,
ZopfliNode *nodes)
{
size_t max_zopfli_len = MaxZopfliLen(params_);
StartPosQueue queue;
size_t cur_match_pos = 0;
size_t i;
nodes[0].length = 0;
nodes[0].u.cost = 0;
InitStartPosQueue(&queue);
for (i = 0; i + 3 < num_bytes; i++)
{
size_t skip = UpdateNodes(num_bytes, position, i, ringbuffer,
ringbuffer_mask, params_, max_backward_limit, dist_cache,
num_matches[i], &matches[cur_match_pos], model, &queue, nodes);
if (skip < BROTLI_LONG_COPY_QUICK_STEP)
{
skip = 0;
}
cur_match_pos += num_matches[i];
if (num_matches[i] == 1 &&
BackwardMatchLength(&matches[cur_match_pos - 1]) > max_zopfli_len)
{
skip = Math.Max(
BackwardMatchLength(&matches[cur_match_pos - 1]), skip);
}
if (skip > 1)
{
skip--;
while (skip != 0)
{
i++;
if (i + 3 >= num_bytes)
{
break;
}
EvaluateNode(
position, i, max_backward_limit, dist_cache, model, &queue, nodes);
cur_match_pos += num_matches[i];
skip--;
}
}
}
return(ComputeShortestPathFromNodes(num_bytes, nodes));
}
/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it
* is eligible. */
private static unsafe void EvaluateNode(
size_t block_start, size_t pos, size_t max_backward_limit,
int *starting_dist_cache, ZopfliCostModel *model,
StartPosQueue *queue, ZopfliNode *nodes)
{
/* Save cost, because ComputeDistanceCache invalidates it. */
float node_cost = nodes[pos].u.cost;
nodes[pos].u.shortcut = ComputeDistanceShortcut(
block_start, pos, max_backward_limit, nodes);
if (node_cost <= ZopfliCostModelGetLiteralCosts(model, 0, pos))
{
PosData posdata;
posdata.pos = pos;
posdata.cost = node_cost;
posdata.costdiff = node_cost -
ZopfliCostModelGetLiteralCosts(model, 0, pos);
ComputeDistanceCache(
pos, starting_dist_cache, nodes, posdata.distance_cache);
StartPosQueuePush(queue, &posdata);
}
}
private static unsafe void ZopfliCostModelSetFromCommands(ZopfliCostModel *self,
size_t position,
byte *ringbuffer,
size_t ringbuffer_mask,
Command *commands,
size_t num_commands,
size_t last_insert_len)
{
uint * histogram_literal = stackalloc uint[BROTLI_NUM_LITERAL_SYMBOLS];
uint * histogram_cmd = stackalloc uint[BROTLI_NUM_COMMAND_SYMBOLS];
uint * histogram_dist = stackalloc uint[BROTLI_NUM_DISTANCE_SYMBOLS];
float *cost_literal = stackalloc float[BROTLI_NUM_LITERAL_SYMBOLS];
size_t pos = position - last_insert_len;
float min_cost_cmd = kInfinity;
size_t i;
float *cost_cmd = self->cost_cmd_;
memset(histogram_literal, 0, BROTLI_NUM_LITERAL_SYMBOLS * sizeof(uint));
memset(histogram_cmd, 0, BROTLI_NUM_COMMAND_SYMBOLS * sizeof(uint));
memset(histogram_dist, 0, BROTLI_NUM_DISTANCE_SYMBOLS * sizeof(uint));
for (i = 0; i < num_commands; i++)
{
size_t inslength = commands[i].insert_len_;
size_t copylength = CommandCopyLen(&commands[i]);
size_t distcode = commands[i].dist_prefix_;
size_t cmdcode = commands[i].cmd_prefix_;
size_t j;
histogram_cmd[cmdcode]++;
if (cmdcode >= 128)
{
histogram_dist[distcode]++;
}
for (j = 0; j < inslength; j++)
{
histogram_literal[ringbuffer[(pos + j) & ringbuffer_mask]]++;
}
pos += inslength + copylength;
}
SetCost(histogram_literal, BROTLI_NUM_LITERAL_SYMBOLS, cost_literal);
SetCost(histogram_cmd, BROTLI_NUM_COMMAND_SYMBOLS, cost_cmd);
SetCost(histogram_dist, BROTLI_NUM_DISTANCE_SYMBOLS, self->cost_dist_);
for (i = 0; i < BROTLI_NUM_COMMAND_SYMBOLS; ++i)
{
min_cost_cmd = Math.Min(min_cost_cmd, cost_cmd[i]);
}
self->min_cost_cmd_ = min_cost_cmd;
{
float *literal_costs = self->literal_costs_;
size_t num_bytes = self->num_bytes_;
literal_costs[0] = 0.0f;
for (i = 0; i < num_bytes; ++i)
{
literal_costs[i + 1] = literal_costs[i] +
cost_literal[ringbuffer[(position + i) & ringbuffer_mask]];
}
}
}
/* Returns longest copy length. */
private static unsafe size_t UpdateNodes(
size_t num_bytes, size_t block_start, size_t pos,
byte *ringbuffer, size_t ringbuffer_mask,
BrotliEncoderParams *params_, size_t max_backward_limit,
int *starting_dist_cache, size_t num_matches,
BackwardMatch *matches, ZopfliCostModel *model,
StartPosQueue *queue, ZopfliNode *nodes)
{
size_t cur_ix = block_start + pos;
size_t cur_ix_masked = cur_ix & ringbuffer_mask;
size_t max_distance = Math.Min(cur_ix, max_backward_limit);
size_t max_len = num_bytes - pos;
size_t max_zopfli_len = MaxZopfliLen(params_);
size_t max_iters = MaxZopfliCandidates(params_);
size_t min_len;
size_t result = 0;
size_t k;
EvaluateNode(block_start, pos, max_backward_limit, starting_dist_cache, model,
queue, nodes);
{
PosData *posdata = StartPosQueueAt(queue, 0);
float min_cost = (posdata->cost + ZopfliCostModelGetMinCostCmd(model) +
ZopfliCostModelGetLiteralCosts(model, posdata->pos, pos));
min_len = ComputeMinimumCopyLength(min_cost, nodes, num_bytes, pos);
}
/* Go over the command starting positions in order of increasing cost
* difference. */
for (k = 0; k < max_iters && k < StartPosQueueSize(queue); ++k)
{
PosData *posdata = StartPosQueueAt(queue, k);
size_t start = posdata->pos;
ushort inscode = GetInsertLengthCode(pos - start);
float start_costdiff = posdata->costdiff;
float base_cost = start_costdiff + (float)GetInsertExtra(inscode) +
ZopfliCostModelGetLiteralCosts(model, 0, pos);
/* Look for last distance matches using the distance cache from this
* starting position. */
size_t best_len = min_len - 1;
size_t j = 0;
for (; j < BROTLI_NUM_DISTANCE_SHORT_CODES && best_len < max_len; ++j)
{
size_t idx = kDistanceCacheIndex[j];
size_t backward =
(size_t)(posdata->distance_cache[idx] + kDistanceCacheOffset[j]);
size_t prev_ix = cur_ix - backward;
if (prev_ix >= cur_ix)
{
continue;
}
if ((backward > max_distance))
{
continue;
}
prev_ix &= ringbuffer_mask;
if (cur_ix_masked + best_len > ringbuffer_mask ||
prev_ix + best_len > ringbuffer_mask ||
ringbuffer[cur_ix_masked + best_len] !=
ringbuffer[prev_ix + best_len])
{
continue;
}
{
size_t len =
FindMatchLengthWithLimit(&ringbuffer[prev_ix],
&ringbuffer[cur_ix_masked],
max_len);
float dist_cost = base_cost +
ZopfliCostModelGetDistanceCost(model, j);
size_t l;
for (l = best_len + 1; l <= len; ++l)
{
ushort copycode = GetCopyLengthCode(l);
ushort cmdcode =
CombineLengthCodes(inscode, copycode, j == 0);
float cost = (cmdcode < 128 ? base_cost : dist_cost) +
(float)GetCopyExtra(copycode) +
ZopfliCostModelGetCommandCost(model, cmdcode);
if (cost < nodes[pos + l].u.cost)
{
UpdateZopfliNode(nodes, pos, start, l, l, backward, j + 1, cost);
result = Math.Max(result, l);
}
best_len = l;
}
}
}
/* At higher iterations look only for new last distance matches, since
* looking only for new command start positions with the same distances
* does not help much. */
if (k >= 2)
{
continue;
}
{
/* Loop through all possible copy lengths at this position. */
size_t len = min_len;
for (j = 0; j < num_matches; ++j)
{
BackwardMatch match = matches[j];
size_t dist = match.distance;
bool is_dictionary_match = (dist > max_distance);
/* We already tried all possible last distance matches, so we can use
* normal distance code here. */
size_t dist_code = dist + BROTLI_NUM_DISTANCE_SHORT_CODES - 1;
ushort dist_symbol;
uint distextra;
uint distnumextra;
float dist_cost;
size_t max_match_len;
PrefixEncodeCopyDistance(dist_code, 0, 0, &dist_symbol, &distextra);
distnumextra = distextra >> 24;
dist_cost = base_cost + (float)distnumextra +
ZopfliCostModelGetDistanceCost(model, dist_symbol);
/* Try all copy lengths up until the maximum copy length corresponding
* to this distance. If the distance refers to the private static unsafe dictionary, or
* the maximum length is long enough, try only one maximum length. */
max_match_len = BackwardMatchLength(&match);
if (len < max_match_len &&
(is_dictionary_match || max_match_len > max_zopfli_len))
{
len = max_match_len;
}
for (; len <= max_match_len; ++len)
{
size_t len_code =
is_dictionary_match ? BackwardMatchLengthCode(&match) : len;
ushort copycode = GetCopyLengthCode(len_code);
ushort cmdcode = CombineLengthCodes(inscode, copycode, false);
float cost = dist_cost + (float)GetCopyExtra(copycode) +
ZopfliCostModelGetCommandCost(model, cmdcode);
if (cost < nodes[pos + len].u.cost)
{
UpdateZopfliNode(nodes, pos, start, len, len_code, dist, 0, cost);
result = Math.Max(result, len);
}
}
}
}
}
return(result);
}
private static unsafe float ZopfliCostModelGetMinCostCmd(
ZopfliCostModel *self)
{
return(self->min_cost_cmd_);
}
private static unsafe float ZopfliCostModelGetLiteralCosts(
ZopfliCostModel *self, size_t from, size_t to)
{
return(self->literal_costs_[to] - self->literal_costs_[from]);
}
private static unsafe float ZopfliCostModelGetDistanceCost(
ZopfliCostModel *self, size_t distcode)
{
return(self->cost_dist_[distcode]);
}
private static unsafe float ZopfliCostModelGetCommandCost(
ZopfliCostModel *self, ushort cmdcode)
{
return(self->cost_cmd_[cmdcode]);
}
private static unsafe void InitZopfliCostModel(
ref MemoryManager m, ZopfliCostModel *self, size_t num_bytes)
{
self->num_bytes_ = num_bytes;
self->literal_costs_ = (float *)BrotliAllocate(ref m, (num_bytes + 2) * sizeof(float));
}
private static unsafe void CleanupZopfliCostModel(ref MemoryManager m, ZopfliCostModel *self)
{
BrotliFree(ref m, self->literal_costs_);
}
