public static void ClusterHistograms(global::Array <object> input, int num_contexts, int num_blocks, int max_histograms, global::Array <object> output, int outputInt, int[] histogram_symbols)
{
unchecked {
int in_size = (num_contexts * num_blocks);
int[] cluster_size = ((int[])(new int[in_size]));
global::DefaultFunctions.memset_Int(cluster_size, 0, 1, in_size);
while ((output.length > in_size))
{
output.pop();
}
{
int _g1 = 0;
while ((_g1 < in_size))
{
output[_g1++] = new global::encode.histogram.Histogram(((int)(outputInt)));
}
}
{
int _g11 = 0;
while ((_g11 < in_size))
{
int i = _g11++;
{
int _g3 = 0;
int _g2 = (((int[])(((global::encode.histogram.Histogram)(input[i])).data_)) as global::System.Array).Length;
while ((_g3 < _g2))
{
int a = _g3++;
((int[])(((global::encode.histogram.Histogram)(output[i])).data_))[a] = ((int[])(((global::encode.histogram.Histogram)(input[i])).data_))[a];
}
}
((global::encode.histogram.Histogram)(output[i])).kDataSize = ((global::encode.histogram.Histogram)(input[i])).kDataSize;
((global::encode.histogram.Histogram)(output[i])).total_count_ = ((global::encode.histogram.Histogram)(input[i])).total_count_;
((global::encode.histogram.Histogram)(output[i])).bit_cost_ = global::encode.Bit_cost.PopulationCost(((global::encode.histogram.Histogram)(input[i])));
((int[])(histogram_symbols))[i] = i;
}
}
int i1 = 0;
while ((i1 < in_size))
{
global::encode.Cluster.HistogramCombine(output, cluster_size, histogram_symbols, i1, ((int)(global::System.Math.Min(((double)((in_size - i1))), ((double)(64))))), max_histograms);
i1 += 64;
}
global::encode.Cluster.HistogramCombine(output, cluster_size, histogram_symbols, 0, in_size, max_histograms);
global::encode.Cluster.HistogramRemap(input, in_size, output, histogram_symbols);
global::encode.Cluster.HistogramReindex(output, histogram_symbols);
}
}
public static void EncodeContextMap(int[] context_map, int num_clusters, global::Array <int> storage_ix, uint[] storage)
{
unchecked {
global::encode.Brotli_bit_stream.StoreVarLenUint8((num_clusters - 1), storage_ix, storage);
if ((num_clusters == 1))
{
return;
}
int[] transformed_symbols = global::encode.Brotli_bit_stream.MoveToFrontTransform(context_map);
global::Array <int> rle_symbols = new global::Array <int>();
global::Array <int> extra_bits = new global::Array <int>();
global::Array <int> max_run_length_prefix = new global::Array <int>(new int[] { 6 });
global::encode.Brotli_bit_stream.RunLengthCodeZeros(transformed_symbols, max_run_length_prefix, rle_symbols, extra_bits);
global::encode.histogram.Histogram symbol_histogram = global::encode.Histogram_functions.HistogramContextMap();
{
int _g1 = 0;
int _g = rle_symbols.length;
while ((_g1 < _g))
{
symbol_histogram.Add1(rle_symbols[_g1++]);
}
}
bool use_rle = (max_run_length_prefix[0] > 0);
global::encode.Write_bits.WriteBits(1, ((use_rle) ? (((uint)(1))) : (((uint)(0)))), storage_ix, storage);
if (use_rle)
{
global::encode.Write_bits.WriteBits(4, ((uint)((max_run_length_prefix[0] - 1))), storage_ix, storage);
}
global::encode.entropy_encode.EntropyCode symbol_code = global::encode.Entropy_encode.EntropyCodeContextMap();
global::DefaultFunctions.memset_UInt(symbol_code.depth_, 0, ((uint)(0)), (((uint[])(symbol_code.depth_)) as global::System.Array).Length);
global::DefaultFunctions.memset_UInt(symbol_code.bits_, 0, ((uint)(0)), (((uint[])(symbol_code.bits_)) as global::System.Array).Length);
global::encode.Brotli_bit_stream.BuildAndStoreHuffmanTree(symbol_histogram.data_, (num_clusters + max_run_length_prefix[0]), symbol_code.depth_, 0, symbol_code.bits_, 0, storage_ix, storage);
{
int _g11 = 0;
int _g2 = rle_symbols.length;
while ((_g11 < _g2))
{
int i = _g11++;
global::encode.Write_bits.WriteBits(((int)(((uint[])(symbol_code.depth_))[rle_symbols[i]])), ((uint[])(symbol_code.bits_))[rle_symbols[i]], storage_ix, storage);
if (((rle_symbols[i] > 0) && (rle_symbols[i] <= max_run_length_prefix[0])))
{
global::encode.Write_bits.WriteBits(rle_symbols[i], ((uint)(extra_bits[i])), storage_ix, storage);
}
}
}
global::encode.Write_bits.WriteBits(1, ((uint)(1)), storage_ix, storage);
}
}
public virtual void AddHistogram(global::encode.histogram.Histogram v)
{
this.total_count_ += v.total_count_;
{
int _g1 = 0;
int _g = this.kDataSize;
while ((_g1 < _g))
{
int i = _g1++;
((int[])(this.data_))[i] += ((int[])(v.data_))[i];
}
}
}
public static void RefineEntropyCodes(int HistogramTypeInt, global::Array <uint> data, int length, int stride, global::Array <object> vec)
{
unchecked {
int iters = (((2 * length) / stride) + 100);
global::Array <uint> seed = new global::Array <uint>(new uint[] { ((uint)(7)) });
iters = (((((iters + vec.length) - 1)) / vec.length) * vec.length);
{
int _g1 = 0;
int _g = iters;
while ((_g1 < _g))
{
global::encode.histogram.Histogram sample = new global::encode.histogram.Histogram(((int)(HistogramTypeInt)));
global::encode.Block_splitter.RandomSample(seed, data, length, stride, sample);
((global::encode.histogram.Histogram)(vec[(_g1++ % vec.length)])).AddHistogram(sample);
}
}
}
}
public static void InitialEntropyCodes(int HistogramTypeInt, global::Array <uint> data, int length, int literals_per_histogram, int max_histograms, int stride, global::Array <object> vec)
{
unchecked {
int total_histograms = ((length / literals_per_histogram) + 1);
if ((total_histograms > max_histograms))
{
total_histograms = max_histograms;
}
int seed_0 = 7;
int block_length = (length / total_histograms);
{
int _g1 = 0;
int _g = total_histograms;
while ((_g1 < _g))
{
int i = _g1++;
int pos = ((length * i) / total_histograms);
if ((i != 0))
{
seed_0 = ((uint)((seed_0 * 16807)));
seed_0 = ((uint)(((uint)((((uint)(seed_0)) >> 0)))));
if (((bool)((seed_0 == 0))))
{
seed_0 = ((uint)(1));
}
pos = ((int)(((uint)((((uint)((seed_0 % block_length))) + pos)))));
}
if (((pos + stride) >= length))
{
pos = ((length - stride) - 1);
}
global::encode.histogram.Histogram histo = new global::encode.histogram.Histogram(((int)(HistogramTypeInt)));
histo.Add2(data, pos, stride);
vec.push(histo);
}
}
}
}
public static void ClusterBlocks(int HistogramTypeInt, global::Array <uint> data, int length, uint[] block_ids)
{
unchecked {
global::Array <object> histograms = new global::Array <object>();
int[] block_index = global::FunctionMalloc.mallocInt(length);
int cur_idx = 0;
global::encode.histogram.Histogram cur_histogram = new global::encode.histogram.Histogram(((int)(HistogramTypeInt)));
{
int _g1 = 0;
while ((_g1 < length))
{
int i = _g1++;
bool block_boundary = (((i + 1) == length) || ((bool)((((uint)(((uint[])(block_ids))[i])) != ((uint)(((uint[])(block_ids))[(i + 1)]))))));
((int[])(block_index))[i] = cur_idx;
cur_histogram.Add1(((int)(data[i])));
if (block_boundary)
{
histograms.push(cur_histogram);
cur_histogram = new global::encode.histogram.Histogram(((int)(HistogramTypeInt)));
++cur_idx;
}
}
}
global::Array <object> clustered_histograms = new global::Array <object>();
int[] histogram_symbols = ((int[])(new int[histograms.length]));
global::encode.Cluster.ClusterHistograms(histograms, 1, histograms.length, 256, clustered_histograms, HistogramTypeInt, histogram_symbols);
{
int _g11 = 0;
while ((_g11 < length))
{
int i1 = _g11++;
((uint[])(block_ids))[i1] = ((uint)(((int)(((int[])(histogram_symbols))[((int)(((int[])(block_index))[i1]))]))));
}
}
}
}
public static double HistogramBitCostDistance(global::encode.histogram.Histogram histogram, global::encode.histogram.Histogram candidate)
{
if ((histogram.total_count_ == 0))
{
return(0.0);
}
global::encode.histogram.Histogram tmp = new global::encode.histogram.Histogram(((int)((((int[])(histogram.data_)) as global::System.Array).Length)));
tmp.bit_cost_ = histogram.bit_cost_;
{
int _g1 = 0;
int _g = (((int[])(histogram.data_)) as global::System.Array).Length;
while ((_g1 < _g))
{
int a = _g1++;
((int[])(tmp.data_))[a] = ((int[])(histogram.data_))[a];
}
}
tmp.kDataSize = histogram.kDataSize;
tmp.total_count_ = histogram.total_count_;
tmp.AddHistogram(candidate);
return(global::encode.Bit_cost.PopulationCost(tmp) - candidate.bit_cost_);
}
public static void __hx_ctor_encode_histogram_Histogram(global::encode.histogram.Histogram __hx_this, int kDataSize)
{
__hx_this.kDataSize = kDataSize;
__hx_this.data_ = ((int[])(new int[kDataSize]));
__hx_this.Clear();
}
public static void RandomSample(global::Array <uint> seed, global::Array <uint> data, int length, int stride, global::encode.histogram.Histogram sample)
{
unchecked {
int pos = 0;
if ((stride >= length))
{
pos = 0;
stride = length;
}
else
{
seed[0] = ((uint)((seed[0] * 16807)));
seed[0] = ((uint)(((uint)((((uint)(seed[0])) >> 0)))));
if (((bool)((seed[0] == 0))))
{
seed[0] = ((uint)(1));
}
pos = ((int)(((uint)((seed[0] % (((length - stride) + 1)))))));
}
sample.Add2(data, pos, stride);
}
}
public static bool StoreMetaBlockTrivial(uint[] input, int start_pos, int length, int mask, bool is_last, global::Array <object> commands, int n_commands, global::Array <int> storage_ix, uint[] storage, int storage_off)
{
unchecked {
if (!(global::encode.Brotli_bit_stream.StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage)))
{
return(false);
}
if ((length == 0))
{
global::encode.Brotli_bit_stream.JumpToByteBoundary(storage_ix, storage);
return(true);
}
global::encode.histogram.Histogram lit_histo = global::encode.Histogram_functions.HistogramLiteral();
global::encode.histogram.Histogram cmd_histo = global::encode.Histogram_functions.HistogramCommand();
global::encode.histogram.Histogram dist_histo = global::encode.Histogram_functions.HistogramDistance();
int pos = start_pos;
{
int _g1 = 0;
while ((_g1 < n_commands))
{
global::encode.command.Command cmd = ((global::encode.command.Command)(commands[_g1++]));
cmd_histo.Add1(((int)(cmd.cmd_prefix_[0])));
{
int _g3 = 0;
int _g2 = cmd.insert_len_;
while ((_g3 < _g2))
{
++_g3;
lit_histo.Add1(((int)(((uint)(((uint[])(input))[(pos & mask)])))));
++pos;
}
}
pos += cmd.copy_len_;
if (((cmd.copy_len_ > 0) && ((bool)((cmd.cmd_prefix_[0] >= 128)))))
{
dist_histo.Add1(((int)(cmd.dist_prefix_[0])));
}
}
}
global::encode.Write_bits.WriteBits(13, ((uint)(0)), storage_ix, storage);
uint[] lit_depth = global::FunctionMalloc.mallocUInt(256);
uint[] lit_bits = global::FunctionMalloc.mallocUInt(256);
uint[] cmd_depth = global::FunctionMalloc.mallocUInt(704);
uint[] cmd_bits = global::FunctionMalloc.mallocUInt(704);
uint[] dist_depth = global::FunctionMalloc.mallocUInt(64);
uint[] dist_bits = global::FunctionMalloc.mallocUInt(64);
global::encode.Brotli_bit_stream.BuildAndStoreHuffmanTree(lit_histo.data_, 256, lit_depth, 0, lit_bits, 0, storage_ix, storage);
global::encode.Brotli_bit_stream.BuildAndStoreHuffmanTree(cmd_histo.data_, 704, cmd_depth, 0, cmd_bits, 0, storage_ix, storage);
global::encode.Brotli_bit_stream.BuildAndStoreHuffmanTree(dist_histo.data_, 64, dist_depth, 0, dist_bits, 0, storage_ix, storage);
pos = start_pos;
{
int _g11 = 0;
while ((_g11 < n_commands))
{
global::encode.command.Command cmd1 = ((global::encode.command.Command)(commands[_g11++]));
int cmd_code = ((int)(cmd1.cmd_prefix_[0]));
int lennumextra = ((int)(((uint)(((uint)((((uint)(cmd1.cmd_extra_[0])) >> 16)))))));
global::Array <uint> lenextra = cmd1.cmd_extra_;
global::encode.Write_bits.WriteBits(((int)(((uint)(((uint[])(cmd_depth))[cmd_code])))), ((uint)(((uint[])(cmd_bits))[cmd_code])), storage_ix, storage);
if ((lennumextra >= 32))
{
global::encode.Write_bits.WriteBits((lennumextra - 32), lenextra[0], storage_ix, storage);
}
global::encode.Write_bits.WriteBits((((lennumextra < 32)) ? (lennumextra) : (32)), lenextra[1], storage_ix, storage);
{
int _g31 = 0;
int _g21 = cmd1.insert_len_;
while ((_g31 < _g21))
{
++_g31;
uint literal = ((uint)(((uint[])(input))[(pos & mask)]));
global::encode.Write_bits.WriteBits(((int)(((uint)(((uint[])(lit_depth))[((int)(literal))])))), ((uint)(((uint[])(lit_bits))[((int)(literal))])), storage_ix, storage);
++pos;
}
}
pos += cmd1.copy_len_;
if (((cmd1.copy_len_ > 0) && ((bool)((cmd1.cmd_prefix_[0] >= 128)))))
{
int dist_code = ((int)(cmd1.dist_prefix_[0]));
int distnumextra = ((int)(((uint)(((uint)((((uint)(cmd1.dist_extra_[0])) >> 24)))))));
int distextra = ((int)(((uint)((cmd1.dist_extra_[0] & 16777215)))));
global::encode.Write_bits.WriteBits(((int)(((uint)(((uint[])(dist_depth))[dist_code])))), ((uint)(((uint[])(dist_bits))[dist_code])), storage_ix, storage);
global::encode.Write_bits.WriteBits(distnumextra, ((uint)(distextra)), storage_ix, storage);
}
}
}
if (is_last)
{
global::encode.Brotli_bit_stream.JumpToByteBoundary(storage_ix, storage);
}
return(true);
}
}
public static void CompareAndPushToHeap(global::Array <object> @out, int[] cluster_size, int idx1, int idx2, global::encode.BinaryHeap <object> pairs)
{
if ((idx1 == idx2))
{
return;
}
if ((idx2 < idx1))
{
int t = idx2;
idx2 = idx1;
idx1 = t;
}
bool store_pair = false;
global::encode.cluster.HistogramPair p = new global::encode.cluster.HistogramPair();
p.idx1 = idx1;
p.idx2 = idx2;
p.valid = true;
p.cost_diff = (0.5 * global::encode.Cluster.ClusterCostDiff(((int)(((int[])(cluster_size))[idx1])), ((int)(((int[])(cluster_size))[idx2]))));
p.cost_diff = ((p.cost_diff -= ((global::encode.histogram.Histogram)(@out[idx1])).bit_cost_) - ((global::encode.histogram.Histogram)(@out[idx2])).bit_cost_);
if ((((global::encode.histogram.Histogram)(@out[idx1])).total_count_ == 0))
{
p.cost_combo = ((global::encode.histogram.Histogram)(@out[idx2])).bit_cost_;
store_pair = true;
}
else if ((((global::encode.histogram.Histogram)(@out[idx2])).total_count_ == 0))
{
p.cost_combo = ((global::encode.histogram.Histogram)(@out[idx1])).bit_cost_;
store_pair = true;
}
else
{
double threshold = (((pairs.size() == 0)) ? (1e99) : (global::System.Math.Max(((double)(0.0)), ((double)(global::haxe.lang.Runtime.toDouble(global::haxe.lang.Runtime.getField(pairs.arr[0], "cost_diff", 1698677367, true)))))));
global::encode.histogram.Histogram combo = new global::encode.histogram.Histogram(((int)((((int[])(((global::encode.histogram.Histogram)(@out[idx1])).data_)) as global::System.Array).Length)));
combo.bit_cost_ = ((global::encode.histogram.Histogram)(@out[idx1])).bit_cost_;
{
int _g1 = 0;
int _g = (((int[])(((global::encode.histogram.Histogram)(@out[idx1])).data_)) as global::System.Array).Length;
while ((_g1 < _g))
{
int a = _g1++;
((int[])(combo.data_))[a] = ((int[])(((global::encode.histogram.Histogram)(@out[idx1])).data_))[a];
}
}
combo.kDataSize = ((global::encode.histogram.Histogram)(@out[idx1])).kDataSize;
combo.total_count_ = ((global::encode.histogram.Histogram)(@out[idx1])).total_count_;
combo.AddHistogram(((global::encode.histogram.Histogram)(@out[idx2])));
double cost_combo = global::encode.Bit_cost.PopulationCost(combo);
if ((cost_combo < (threshold - p.cost_diff)))
{
p.cost_combo = cost_combo;
store_pair = true;
}
}
if (store_pair)
{
p.cost_diff += p.cost_combo;
pairs.push(p);
}
}
public static void HistogramReindex(global::Array <object> @out, int[] symbols)
{
global::Array <object> tmp = new global::Array <object>();
{
int _g1 = 0;
int _g = @out.length;
while ((_g1 < _g))
{
int i = _g1++;
tmp[i] = new global::encode.histogram.Histogram(((int)((((int[])(((global::encode.histogram.Histogram)(@out[i])).data_)) as global::System.Array).Length)));
((global::encode.histogram.Histogram)(tmp[i])).bit_cost_ = ((global::encode.histogram.Histogram)(@out[i])).bit_cost_;
{
int _g3 = 0;
int _g2 = (((int[])(((global::encode.histogram.Histogram)(@out[i])).data_)) as global::System.Array).Length;
while ((_g3 < _g2))
{
int a = _g3++;
((int[])(((global::encode.histogram.Histogram)(tmp[i])).data_))[a] = ((int[])(((global::encode.histogram.Histogram)(@out[i])).data_))[a];
}
}
((global::encode.histogram.Histogram)(tmp[i])).kDataSize = ((global::encode.histogram.Histogram)(@out[i])).kDataSize;
((global::encode.histogram.Histogram)(tmp[i])).total_count_ = ((global::encode.histogram.Histogram)(@out[i])).total_count_;
}
}
global::haxe.ds.IntMap <int> new_index = new global::haxe.ds.IntMap <int>();
int next_index = 0;
{
int _g11 = 0;
int _g4 = (((int[])(symbols)) as global::System.Array).Length;
while ((_g11 < _g4))
{
int i1 = _g11++;
if ((new_index.exists(((int)(((int[])(symbols))[i1]))) == false))
{
new_index.@set(((int)(((int[])(symbols))[i1])), next_index);
((global::encode.histogram.Histogram)(@out[next_index])).bit_cost_ = ((global::encode.histogram.Histogram)(tmp[((int)(((int[])(symbols))[i1]))])).bit_cost_;
{
int _g31 = 0;
int _g21 = (((int[])(((global::encode.histogram.Histogram)(tmp[((int)(((int[])(symbols))[i1]))])).data_)) as global::System.Array).Length;
while ((_g31 < _g21))
{
int a1 = _g31++;
((int[])(((global::encode.histogram.Histogram)(@out[next_index])).data_))[a1] = ((int[])(((global::encode.histogram.Histogram)(tmp[((int)(((int[])(symbols))[i1]))])).data_))[a1];
}
}
((global::encode.histogram.Histogram)(@out[next_index])).kDataSize = ((global::encode.histogram.Histogram)(tmp[((int)(((int[])(symbols))[i1]))])).kDataSize;
((global::encode.histogram.Histogram)(@out[next_index])).total_count_ = ((global::encode.histogram.Histogram)(tmp[((int)(((int[])(symbols))[i1]))])).total_count_;
++next_index;
}
}
}
while ((@out.length > next_index))
{
@out.pop();
}
{
int _g12 = 0;
int _g5 = (((int[])(symbols)) as global::System.Array).Length;
while ((_g12 < _g5))
{
int i2 = _g12++;
((int[])(symbols))[i2] = (new_index.@get(((int)(((int[])(symbols))[i2])))).@value;
}
}
}
public static double PopulationCost(global::encode.histogram.Histogram histogram)
{
unchecked {
int kSize = (((int[])(histogram.data_)) as global::System.Array).Length;
if ((histogram.total_count_ == 0))
{
return((double)(12));
}
int count = 0;
{
int _g1 = 0;
while ((_g1 < kSize))
{
if ((((int[])(histogram.data_))[_g1++] > 0))
{
++count;
}
}
}
if ((count == 1))
{
return((double)(12));
}
if ((count == 2))
{
return((double)((20 + histogram.total_count_)));
}
double bits = ((double)(0));
uint[] depth = global::FunctionMalloc.mallocUInt(kSize);
if ((count <= 4))
{
global::encode.Entropy_encode.CreateHuffmanTree(histogram.data_, 0, kSize, 15, depth, 0);
{
int _g11 = 0;
while ((_g11 < kSize))
{
int i = _g11++;
bits = ((double)((((uint)((((uint)(((uint[])(depth))[i])) * ((int[])(histogram.data_))[i]))) + bits)));
}
}
if ((count == 3))
{
return(bits + 28);
}
else
{
return(bits + 37);
}
}
int max_depth = 1;
int[] depth_histo = global::FunctionMalloc.mallocInt(18);
double log2total = global::encode.Fast_log.FastLog2(histogram.total_count_);
int i1 = 0;
while ((i1 < kSize))
{
if ((((int[])(histogram.data_))[i1] > 0))
{
double log2p = (log2total - global::encode.Fast_log.FastLog2(((int[])(histogram.data_))[i1]));
int depth1 = ((int)((log2p + 0.5)));
bits += (((int[])(histogram.data_))[i1] * log2p);
if ((depth1 > 15))
{
depth1 = 15;
}
if ((depth1 > max_depth))
{
max_depth = depth1;
}
{
int _g = depth1;
((int[])(depth_histo))[_g] = (((int)(((int[])(depth_histo))[_g])) + 1);
}
++i1;
}
else
{
int reps = 1;
int k = (i1 + 1);
while (((k < kSize) && (((int[])(histogram.data_))[k] == 0)))
{
++reps;
++k;
}
i1 += reps;
if ((i1 == kSize))
{
break;
}
if ((reps < 3))
{
((int[])(depth_histo))[0] = (((int)(((int[])(depth_histo))[0])) + reps);
}
else
{
reps -= 2;
while ((reps > 0))
{
((int[])(depth_histo))[17] = (((int)(((int[])(depth_histo))[17])) + 1);
bits += ((double)(3));
reps >>= 3;
}
}
}
}
bits += ((double)((18 + (2 * max_depth))));
bits += global::encode.Bit_cost.BitsEntropy(depth_histo, 0, 18);
return(bits);
}
}
