public override void WriteHeader(Bitstream bitstream, Datastream data, Header.HeaderFlags headerFlags)
{
Header.WriteUniversalHeader(bitstream, data, headerFlags);
if (data.Count == 0)
{
return;
}
// make code tree
var tree = MakeTree(data);
// walk tree assigning codewords
AssignBitStrings(tree);
// get leaf nodes, which are the symbols
// after this the rest of the tree is not needed
leaves.Clear();
GetLeaves(tree);
MakeCanonical(leaves); // relabel codewords into canonical ordering
// write symbol table
// must have canonical labeled leaves
bitstream.WriteStream(MakeTable(leaves));
// prepare to add rest of data
if (Options.HasFlag(OptionFlags.DumpEncoding))
{
WriteLine("Huff encode: [");
}
}
/// <summary>
/// Compress stream of numbers:
/// Store length+1 using EliasDelta (to avoid 0 case)
/// First number x0 requires b0 bits. Write b0 in EliasDelta. Write x0 in b0 bits.
/// Each subsequent number xi requires bi bits. If bi leq b(i-1) then
/// write a 0 bit, then xi in b(i-1) bits. Else write (bi-b(i-1)) 1 bits, then a 0,
/// then the least sig bi - 1 bits of xi (the leading 1 in xi is implied, xi>0).
/// TODO - alternatives - allow the bi to change slowly, removes some hiccups for odd data points, set b to avg of some prev values
/// </summary>
/// <param name="bitstream"></param>
/// <param name="data"></param>
/// <param name="universalCoder">How to encode/decode a data length and first bitlength items. Elias.EncodeDelta is useful</param>
public static void BinaryAdaptiveSequentialEncode(Bitstream bitstream, Datastream data, Action <Bitstream, uint> universalCoder)
{
universalCoder(bitstream, (uint)data.Count + 1);
if (data.Count == 0)
{
return;
}
uint b1 = CodecBase.BitsRequired(data[0]);
universalCoder(bitstream, b1);
bitstream.Write(data[0]);
for (var i = 1; i < data.Count; ++i)
{
uint d = data[i];
uint b2 = CodecBase.BitsRequired(d);
if (b2 <= b1)
{ // b1 is enough bits
bitstream.Write(0);
bitstream.Write(d, b1);
}
else
{ // b2 requires more bits, tell how many
Trace.Assert(d > 0);
for (var ik = 0; ik < b2 - b1; ++ik)
{
bitstream.Write(1);
}
bitstream.Write(0, 1); // end of bit count
bitstream.Write(d, b2 - 1); // strip off leading '1'
}
b1 = CodecBase.BitsRequired(d); // for next pass
}
}
// end encoding notes http://www3.sympatico.ca/mt0000/biacode/biacode.html
// and http://bijective.dogma.net/compres10.htm
/// <summary>
/// Write the header for the compression algorithm
/// </summary>
/// <param name="bitstream"></param>
/// <param name="data"></param>
/// <param name="headerFlags">Flags telling what to put in the header. Useful when embedding in other streams.</param>
/// <returns></returns>
public override void WriteHeader(Bitstream bitstream, Datastream data, Header.HeaderFlags headerFlags)
{
// count occurrences to get probabilities
counts = new uint[data.Max() + 1];
foreach (var b in data)
{
counts[b]++;
}
total = (uint)data.Count; // total frequency
MakeSums();
ResetCoder();
// arithmetic gets total probability from the header, so ensure it gets saved
headerFlags |= Header.HeaderFlags.SymbolCount;
Header.WriteUniversalHeader(bitstream, data, headerFlags);
// we'll insert the bitlength of what follows at this spot during the footer
whereToInsertBitlength = bitstream.Position;
// write freq tables
var tables = MakeFrequencyTable();
bitstream.WriteStream(tables);
if (Options.HasFlag(OptionFlags.DumpState))
{
Write("Enc: ");
}
}
/// <summary>
/// Make probability tree
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
Node MakeTree(Datastream data)
{
// create a node for each symbol with the frequency count
var nodes = new List <Node>();
foreach (var symbol in data)
{
AddNode(symbol, nodes);
}
// create a parent for the lowest two freq nodes until single root node left
while (nodes.Count > 1)
{
// get lowest freq node
var minFreqL = nodes.Min(n => n.FrequencyCount);
var left = nodes.First(n => n.FrequencyCount == minFreqL);
nodes.Remove(left);
// get second lowest freq node
var minFreqR = nodes.Min(n => n.FrequencyCount);
var right = nodes.First(n => n.FrequencyCount == minFreqR);
nodes.Remove(right);
// combine and reinsert
nodes.Add(new Node
{
LeftChild = left,
RightChild = right,
FrequencyCount = left.FrequencyCount + right.FrequencyCount
});
}
return(nodes[0]);
}
/// <summary>
/// try various compression on the data,
/// return the best codec and compute the bits saved by it
/// </summary>
/// <returns></returns>
private Tuple <Type, Bitstream> GetBestCompressor(
string label,
List <uint> data
)
{
// use this to check each stream
var cc = new CompressionChecker {
Options = CompressionOptions
};
var stream = new Datastream(data);
var results = cc.TestAll(label, stream, internalFlags);
results.Sort((a, b) => a.CompressedBitLength.CompareTo(b.CompressedBitLength));
var best = results[0];
CodecBase codec;
if (best.CompressorType == typeof(FixedSizeCodec))
{
codec = new FixedSizeCodec();
}
else if (best.CompressorType == typeof(ArithmeticCodec))
{
codec = new ArithmeticCodec();
}
else if (best.CompressorType == typeof(HuffmanCodec))
{
codec = new HuffmanCodec();
}
else if (best.CompressorType == typeof(GolombCodec))
{
codec = new GolombCodec {
Parameter = best.Parameters[0]
}
}
;
else
{
throw new NotImplementedException("Unknown codec type");
}
var bitstream = codec.CompressToStream(stream, internalFlags);
var codecName = codec.GetType().Name;
StatRecorder.AddStat("codec win: " + label + " " + codecName, 1);
StatRecorder.AddStat($"codec win {codecName} saved high ", results.Last().CompressedBitLength - best.CompressedBitLength);
if (results.Count > 1)
{
StatRecorder.AddStat($"codec win {codecName} saved low ", results[1].CompressedBitLength - best.CompressedBitLength);
}
if (Options.HasFlag(OptionFlags.DumpCompressorSelections))
{
WriteLine($"{label} using {codecName}");
}
return(new Tuple <Type, Bitstream>(codec.GetType(), bitstream));
}
/// <summary>
/// Write the header for the compression algorithm
/// </summary>
/// <param name="bitstream"></param>
/// <param name="data"></param>
/// <param name="headerFlags">Flags telling what to put in the header. Useful when embedding in other streams.</param>
/// <returns></returns>
public override void WriteHeader(Bitstream bitstream, Datastream data, Header.HeaderFlags headerFlags)
{
// always store bps also
Header.WriteUniversalHeader(bitstream, data, headerFlags | Header.HeaderFlags.BitsPerSymbol);
var max = data.Any() ? data.Max() : 0;
BitsPerSymbol = BitsRequired(max);
}
/// <summary>
/// replace each entry with the diff from the prev one
/// first one is left alone. Return deltas
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
public static Datastream DeltaCode(Datastream data)
{
var d = new Datastream(data);
for (var i = data.Count - 1; i >= 1; --i)
{
d[i] = (byte)(data[i] - data[i - 1]);
}
d[0] = data[0];
return(d);
}
/// <summary>
/// Stream version
/// </summary>
/// <param name="bitstream"></param>
/// <param name="headerFlags"></param>
/// <returns></returns>
public Datastream DecompressFromStream(Bitstream bitstream, Header.HeaderFlags headerFlags)
{
uint symbolCount = ReadHeader(bitstream, headerFlags);
var datastream = new Datastream();
for (var i = 0U; i < symbolCount; ++i)
{
datastream.Add(DecompressSymbol(bitstream));
}
ReadFooter(bitstream);
return(datastream);
}
/// <summary>
/// Process the data to compress, creating various list of values to be compressed later
/// </summary>
/// <param name="data"></param>
private void ScanData(Datastream data)
{
uint index = 0; // index of symbol to encode
while (index < data.Count)
{
// find best run
uint bestDistance = 0;
uint bestLength = 0;
// walk backwards to end with smallest distance for a given run length
for (int distance = (int)MaximumDistance; distance >= 0; --distance)
{
uint length = 0;
while (
// bound check the values
0 <= index - 1 - distance + length &&
index - 1 - distance + length < data.Count &&
index + length < data.Count &&
// check the bytes match
data[(int)(index - 1 - distance + length)] == data[(int)(index + length)] &&
// don't get longer than this
length < MaximumLength
)
{
length++;
}
// keep best
if (length >= bestLength)
{
// break ties in this direction to have shorter distances
bestLength = length;
bestDistance = (uint)distance;
}
}
// now with best run, decide how to encode next part
if (bestLength >= MinimumLength)
{
// encode (distance,length) token
decisions.Add(1); // select (distance,length)
distances.Add(bestDistance);
lengths.Add(bestLength);
index += bestLength;
}
else
{
// encode literal
decisions.Add(0); // select literal
literals.Add(data[(int)index]); // save value
index++;
}
}
}
/// <summary>
/// Stream version to keep from 0 padding last byte
/// </summary>
/// <param name="data"></param>
/// <param name="headerFlags">Flags telling what to put in the header. Useful when embedding in other streams.</param>
/// <returns></returns>
public Bitstream CompressToStream(Datastream data, Header.HeaderFlags headerFlags)
{
var bitstream = new Bitstream();
WriteHeader(bitstream, data, headerFlags);
foreach (var symbol in data)
{
CompressSymbol(bitstream, symbol);
}
WriteFooter(bitstream);
return(bitstream);
}
/// <summary>
/// Write a universal compression header
/// </summary>
/// <param name="bitstream"></param>
/// <param name="data"></param>
/// <param name="headerFlags"></param>
public static void WriteUniversalHeader(Bitstream bitstream, Datastream data, HeaderFlags headerFlags)
{
if (headerFlags.HasFlag(HeaderFlags.SymbolCount))
{
UniversalCodec.Lomont.EncodeLomont1(bitstream, (uint)data.Count, 6, 0);
}
if (headerFlags.HasFlag(HeaderFlags.BitsPerSymbol))
{
var max = data.Any() ? data.Max() : 0;
var bitsPerSymbol = CodecBase.BitsRequired(max);
UniversalCodec.Lomont.EncodeLomont1(bitstream, bitsPerSymbol - 1, 3, 0);
}
}
/// <summary>
/// Compress the data in the streams.
/// </summary>
/// <returns></returns>
private void PackHeader(Bitstream bitstream, Datastream data, Header.HeaderFlags headerFlags)
{
// actual value extremes occurring in data streams
encoderState.ActualMaxDistance = distances.Any() ? distances.Max() : 0;
encoderState.ActualMinLength = lengths.Any() ? lengths.Min() : 0;
// token (distance,length) is encoded as
// (length - actualMinLength) * (actualMaxDistance+1) + distance
// compute largest occurring token value
uint actualMaxToken = 0;
for (var i = 0; i < distances.Count; ++i)
{
uint length = lengths[i];
uint distance = distances[i];
uint token = (length - encoderState.ActualMinLength) * (encoderState.ActualMaxDistance + 1) + distance;
actualMaxToken = Math.Max(actualMaxToken, token);
}
// bit sizes
encoderState.ActualBitsPerSymbol = BitsRequired(literals.Max());
encoderState.ActualBitsPerToken = BitsRequired(actualMaxToken);
//some info to help analyze
if (Options.HasFlag(OptionFlags.DumpHeader))
{
//WriteLine("LZ77 compress:");
//WriteLine($" Data length {data.Count} ");
//WriteLine($" Bits per symbol {encoderState.actualBitsPerSymbol} ");
//WriteLine($" Bits per token {encoderState.actualBitsPerToken}");
//WriteLine($" Max token {actualMaxToken}, {lengths.Count} tokens ");
//WriteLine($" Max distance {encoderState.actualMaxDistance} ");
}
// checks
Trace.Assert(0 < encoderState.ActualBitsPerSymbol);
// header values
Header.WriteUniversalHeader(bitstream, data, headerFlags);
UniversalCodec.Lomont.EncodeLomont1(bitstream, encoderState.ActualBitsPerSymbol - 1, 3, 0); // usually 8, -1 => 7, fits in 3 bits
UniversalCodec.Lomont.EncodeLomont1(bitstream, encoderState.ActualBitsPerToken - 1, 5, 0); // around 20 or so, depends on parameters
UniversalCodec.Lomont.EncodeLomont1(bitstream, encoderState.ActualMinLength, 2, 0); // usually 2
UniversalCodec.Lomont.EncodeLomont1(bitstream, actualMaxToken, 25, -10); //
UniversalCodec.Lomont.EncodeLomont1(bitstream, encoderState.ActualMaxDistance, 14, -7); //
if (Options.HasFlag(OptionFlags.DumpHeader))
{
WriteLine("LZ77 header:");
WriteLine($" datalen {data.Count} bits/symbol {encoderState.ActualBitsPerSymbol}, bits/token {encoderState.ActualBitsPerToken}, minLen {encoderState.ActualMinLength}, max token {actualMaxToken}, max dist {encoderState.ActualMaxDistance}");
}
}
public override void WriteHeader(Bitstream bitstream, Datastream data, Header.HeaderFlags headerFlags)
{
Header.WriteUniversalHeader(bitstream, data, headerFlags);
if (Options.HasFlag(OptionFlags.Optimize))
{
Parameter = Optimize(data, headerFlags);
}
if (Options.HasFlag(OptionFlags.DumpDebug))
{
WriteLine($"Golomb encode parameter {Parameter}");
}
//Console.WriteLine("Parameter " + Parameter);
UniversalCodec.Lomont.EncodeLomont1(bitstream, Parameter, Lomont1Parameter, 0);
}
/// <summary>
/// Write the header for the compression algorithm
/// </summary>
/// <param name="bitstream"></param>
/// <param name="data"></param>
/// <param name="headerFlags">Flags telling what to put in the header. Useful when embedding in other streams.</param>
/// <returns></returns>
public override void WriteHeader(Bitstream bitstream, Datastream data, Header.HeaderFlags headerFlags)
{
// erase data streams
decisions.Clear();
literals.Clear();
distances.Clear();
lengths.Clear();
// fill in various data streams
ScanData(data);
// put header into stream
PackHeader(bitstream, data, headerFlags);
// start indices here
encoderState.Reset();
if (Options.HasFlag(OptionFlags.DumpEncode))
{
Write("LZ77 stream: ");
}
}
/// <summary>
/// try various compression on the data,
/// return list of compression results (bitLength, type, optional parameters)
/// </summary>
/// <param name="statPrefix"></param>
/// <param name="data"></param>
/// <param name="headerFlags"></param>
/// <returns></returns>
public List <Result> TestAll(string statPrefix, Datastream data, Header.HeaderFlags headerFlags)
{
var results = new List <Result>();
// perform compression algorithm
Action <string, CodecBase, Type> tryCodec = (label, codec, codecType) =>
{
var bitstream = codec.CompressToStream(data, headerFlags);
var result = new Result(label, bitstream.Length, codecType);
if (codec.GetType() == typeof(GolombCodec))
{
var g = codec as GolombCodec;
if (g != null)
{
result.Parameters.Add(g.Parameter);
result.CompressorName += $"({g.Parameter})";
}
}
results.Add(result);
};
// try compression algorithms
if (Options.HasFlag(OptionFlags.UseFixed))
{
tryCodec("Fixed size", new FixedSizeCodec(), typeof(FixedSizeCodec));
}
if (Options.HasFlag(OptionFlags.UseArithmetic))
{
tryCodec("Arithmetic", new ArithmeticCodec(), typeof(ArithmeticCodec));
}
if (Options.HasFlag(OptionFlags.UseHuffman))
{
tryCodec("Huffman", new HuffmanCodec(), typeof(HuffmanCodec));
}
if (Options.HasFlag(OptionFlags.UseGolomb) && data.Max() < GolombCodec.GolombThreshold)
{
tryCodec("Golomb", new GolombCodec(), typeof(GolombCodec));
}
/* // try Golomb encoding
* if (Options.HasFlag(OptionFlags.UseGolomb))
* {
* var bestg = UniversalCodec.Optimize(UniversalCodec.Golomb.Encode, data, 1, Math.Min(data.Max(), 256));
* var bitstream = bestg.Item1;
* var gname = $"Golomb({bestg.Item2,2})";
* results.Add(new Result(gname, bitstream.Length, typeof(GolombCodec), bestg.Item2));
* //results.Add(new Result(gname,bitstream.Length, typeof(UniversalCodec.Golomb), bestg.Item2));
* } */
Action <string, UniversalCodec.UniversalCodeDelegate, Type> tryEncoder = (label, codec, codecType) =>
{
var bitstream = UniversalCodec.CompressStream(codec, data.Select(v => v + 1).ToList());
results.Add(new Result(label, bitstream.Length, codecType));
};
// try Elias codes - all perform poorly - todo - need way to pass this back as type?
if (Options.HasFlag(OptionFlags.UseEliasDelta))
{
tryEncoder("EliasDelta", UniversalCodec.Elias.EncodeDelta, typeof(UniversalCodec.Elias));
}
if (Options.HasFlag(OptionFlags.UseEliasGamma))
{
tryEncoder("EliasGamma", UniversalCodec.Elias.EncodeGamma, typeof(UniversalCodec.Elias));
}
if (Options.HasFlag(OptionFlags.UseEliasOmega))
{
tryEncoder("EliasOmega", UniversalCodec.Elias.EncodeOmega, typeof(UniversalCodec.Elias));
}
// Stout
if (Options.HasFlag(OptionFlags.UseStout))
{
tryEncoder("Stout", (b, v) => UniversalCodec.Stout.Encode(b, v, 3), typeof(UniversalCodec.Stout));
}
// BinaryAdaptiveSequentialEncode
if (Options.HasFlag(OptionFlags.UseBasc))
{
var bitstream = new Bitstream();
UniversalCodec.BinaryAdaptiveSequentialEncode(bitstream, data, UniversalCodec.Elias.EncodeDelta);
var label = "BASC";
results.Add(new Result(label, bitstream.Length, typeof(UniversalCodec)));
}
// save stats
foreach (var result in results)
{
StatRecorder.AddStat(statPrefix + "_" + result.CompressorName, result.CompressedBitLength);
}
return(results);
}
/// <summary>
/// Write the header for the compression algorithm
/// </summary>
/// <param name="bitstream"></param>
/// <param name="data"></param>
/// <param name="headerFlags"></param>
/// <returns></returns>
public virtual void WriteHeader(Bitstream bitstream, Datastream data, Header.HeaderFlags headerFlags)
{
}
/// <summary>
/// Process the data to compress, creating various list of values to be compressed later
/// </summary>
/// <param name="data"></param>
/// <param name="actualMinLength"></param>
/// <param name="actualMaxDistance"></param>
private void ComputeStreams(Datastream data, out uint actualMinLength, out uint actualMaxDistance)
{
uint index = 0; // index of symbol to encode
while (index < data.Count)
{
// find best run
uint bestDistance = 0;
uint bestLength = 0;
// walk backwards to end with smallest distance for a given run length
for (int distance = (int)MaximumDistance; distance >= 0; --distance)
{
uint length = 0;
while (
// bound check the values
0 <= index - 1 - distance + length &&
index - 1 - distance + length < data.Count &&
index + length < data.Count &&
// check the bytes match
data[(int)(index - 1 - distance + length)] == data[(int)(index + length)] &&
// don't get longer than this
length < MaximumLength
)
{
length++;
}
// keep best
if (length >= bestLength)
{
// break ties in this direction to have shorter distances
bestLength = length;
bestDistance = (uint)distance;
}
}
// now with best run, decide how to encode next part
if (bestLength >= MinimumLength)
{
// encode (distance,length) token
decisions.Add(1); // select (distance,length)
distances.Add(bestDistance);
lengths.Add(bestLength);
index += bestLength;
}
else
{
// encode literal
decisions.Add(0); // select literal
literals.Add(data[(int)(index)]); // save value
index++;
}
}
Trace.Assert(decisions.Max() < 2);
// some actually occurring extremes
actualMaxDistance = distances.Any()?distances.Max():0;
actualMinLength = lengths.Any()?lengths.Min():0;
// shift lengths
for (var i = 0; i < lengths.Count; ++i)
{
lengths[i] -= actualMinLength;
}
// create tokens in case we need them
for (var i = 0; i < lengths.Count; ++i)
{
tokens.Add(lengths[i] * (actualMaxDistance + 1) + distances[i]);
}
// get runs of each type of decision in case we need them
decisionRuns.AddRange(GetRuns(decisions));
if (Options.HasFlag(OptionFlags.ShowTallies))
{
Write("Decision runs tally ");
Tally(decisionRuns);
WriteLine();
}
}
/// <summary>
/// Create the frequency table as a bitsteam for ease of use/testing
/// </summary>
/// <returns></returns>
Bitstream MakeFrequencyTable()
{
var bs = new Bitstream();
// write freq tables
uint maxCount = counts.Max();
uint minCount = counts.Where(c => c > 0).Min();
#if true
// have determined the following:
// Of all three Elias, Golomb optimized, BASC, that BASC is slightly best for storing counts
// Also using BASC for counts present is good.
// Also determined the sparse table type is much bigger in every file we tested!
// so, check two types:
// 1) BASC on all counts, versus
// 2) BASC on those present for both count and symbol
// Table thus only full type. Format is
// - symbol min index used, max index used, Lomont1 universal coded.
// - Number of bits in table, Lomont1 universal coded (allows jumping past)
// - Full table. Counts are BASC encoded, maxIndex - minIndex+1 entries
uint minSymbolIndex = UInt32.MaxValue;
uint maxSymbolIndex = 0;
for (var i = 0U; i < counts.Length; ++i)
{
if (counts[i] != 0)
{
maxSymbolIndex = i;
if (minSymbolIndex == UInt32.MaxValue)
{
minSymbolIndex = i;
}
}
}
UniversalCodec.Lomont.EncodeLomont1(bs, minSymbolIndex, 6, 0);
UniversalCodec.Lomont.EncodeLomont1(bs, maxSymbolIndex, 6, 0);
var fullTableBs = new Bitstream();
UniversalCodec.BinaryAdaptiveSequentialEncode(fullTableBs, new Datastream(
counts.Skip((int)minSymbolIndex).Take((int)(maxSymbolIndex - minSymbolIndex + 1)).ToArray()),
(b, v) => UniversalCodec.Lomont.EncodeLomont1(b, v, 6, 0)
);
UniversalCodec.Lomont.EncodeLomont1(bs, fullTableBs.Length, 6, 0);
bs.WriteStream(fullTableBs);
if (Options.HasFlag(OptionFlags.DumpHeader))
{
WriteLine($"Arith encode: min symb index {minSymbolIndex} max symb index {maxSymbolIndex} tbl bits {fullTableBs.Length}");
}
#else
// have determined the following:
// Of all three Elias, Golomb optimized, BASC, that BASC is slightly best for storing counts
// Also using BASC for counts present is good.
// Also determined the sparse table type is much bigger in every file we tested!
// so, check two types:
// 1) BASC on all counts, versus
// 2) BASC on those present for both count and symbol
// Table thus
// - symbol min index used + 1, max index used + 1, EliasDelta coded.
// - bit denoting table type 0 (full) or 1 (sparse)
// - Number of bits in table + 1, elias delta coded (allows jumping past)
// 0 = Full table. Counts are BASC encoded, maxIndex - minIndex+1 entries
// 1 = sparse table.
// Elias delta for number of counts in table + 1 (same as number of symbols)
// Elias delta for bitlength of counts + 1,
// BASC counts,
// BASC symbols present
// - table
// compute two table lengths:
uint minSymbolIndex = UInt32.MaxValue;
uint maxSymbolIndex = 0;
for (var i = 0U; i < counts.Length; ++i)
{
if (counts[i] != 0)
{
maxSymbolIndex = i;
if (minSymbolIndex == UInt32.MaxValue)
{
minSymbolIndex = i;
}
}
}
// common header
UniversalCodec.Elias.EncodeDelta(bs, minSymbolIndex + 1);
UniversalCodec.Elias.EncodeDelta(bs, maxSymbolIndex + 1);
var fullTableBs = new Bitstream();
var sparseTableBs = new Bitstream();
UniversalCodec.BinaryAdaptiveSequentialEncode(fullTableBs, new Datastream(
counts.Skip((int)minSymbolIndex).Take((int)(maxSymbolIndex - minSymbolIndex + 1)).ToArray()
));
var nonzeroCountIndices =
counts.Select((c, n) => new { val = c, pos = n })
.Where(p => p.val > 0)
.Select(p => (uint)p.pos)
.ToArray();
var nonzeroCounts = counts.Where(c => c > 0).ToArray();
UniversalCodec.Elias.EncodeDelta(sparseTableBs, (uint)(nonzeroCounts.Length + 1));
UniversalCodec.Elias.EncodeDelta(sparseTableBs, (uint)(nonzeroCounts.Length + 1));
var tempBs = new Bitstream();
UniversalCodec.BinaryAdaptiveSequentialEncode(tempBs, new Datastream(nonzeroCounts));
uint sparseMidPos = tempBs.Position;
UniversalCodec.Elias.EncodeDelta(sparseTableBs, sparseMidPos + 1);
sparseTableBs.WriteStream(tempBs);
UniversalCodec.BinaryAdaptiveSequentialEncode(sparseTableBs, new Datastream(nonzeroCountIndices));
Console.WriteLine($"Arith full table {fullTableBs.Length} sparse table {sparseTableBs.Length}");
// now finish table
if (fullTableBs.Length < sparseTableBs.Length)
{
bs.Write(0); // full table
UniversalCodec.Elias.EncodeDelta(bs, fullTableBs.Length + 1);
bs.WriteStream(fullTableBs);
}
else
{
bs.Write(1); // sparse table
UniversalCodec.Elias.EncodeDelta(bs, sparseTableBs.Length + 1);
bs.WriteStream(sparseTableBs);
}
// var cc = new CompressionChecker();
// cc.TestAll("arith",new Datastream(counts)); // all
// cc.TestAll("arith",new Datastream(counts.Where(c=>c>0).ToArray())); // nonzero
// BASC wins these tests
//
#if false
var allDs = new Datastream();
var nonzeroDs = new Datastream();
for (var i = 0U; i < counts.Length; ++i)
{
var index = i;//(uint)(counts.Length - 1 - i);
allDs.Add(index);
if (counts[i] != 0)
{
nonzeroDs.Add(index);
}
}
var allBs = new Bitstream();
var nonzeroBs = new Bitstream();
UniversalCodec.BinaryAdaptiveSequentialEncode(allBs, allDs);
UniversalCodec.BinaryAdaptiveSequentialEncode(nonzeroBs, nonzeroDs);
Console.WriteLine($"Arith all {allBs.Length} in ");
Console.WriteLine($"Arith nonzero {nonzeroBs.Length} in ");
//foreach (var c in counts)
// UniversalCodec.OneParameterCodeDelegate(
//var ans = UniversalCodec.Optimize(UniversalCodec.Golomb.Encode,counts.ToList(),1,256);
//bs = ans.Item1;
// 912 gamma
// 918 elias delta
// 988 Omega
// 1152 bits UniversalCodec.BinaryAdaptiveSequentialEncode(bs,new Datastream(counts));
// 1265 best Golomb
#endif
#endif
if (Options.HasFlag(OptionFlags.DumpTable))
{
WriteLine($"Arith table bitsize {bs.Length}, min symbol ? max symbol ? min count {minCount} max count {maxCount}");
for (var i = 0; i < counts.Length; ++i)
{
if (counts[i] != 0)
{
Write($"[{i},{counts[i]}] ");
}
}
WriteLine();
}
return(bs);
}
uint Optimize(Datastream data, Header.HeaderFlags headerFlags)
{
// Golomb bitlength seems to be a convex down function of the parameter:
// Let non-negative integers a1,a2,a3,..aN, parameter M. b=#bits to encode M, = Floor[log_2 M]+1
// golomb code then qi=Floor[ai/M],ri=ai-Mqi, unary encode qi in qi+1 bits, encode ri in b or b-1 bits using
// adaptive code. Then each of these is convex (true?) in M, so sum is convex, so length is convex.
// Want best parameter between M=1 (total unary) and M=max{ai} = total fixed encoding
// for large ai, unary uses lots of bits, so start at high end, 2^k=M >= max{ai}, divide by 2 until stops decreasing,
// then binary search on final range.
// todo - writeup optimal selection as blog post
var g = new GolombCodec();
g.Options &= ~OptionFlags.Optimize; // disable auto optimizer
// function to compute length given the parameter
Func <uint, uint> f = m1 =>
{
g.Parameter = m1;
var bs = g.CompressToStream(data, headerFlags);
var len1 = bs.Length;
return(len1);
};
Trace.Assert(data.Max() < 0x80000000); // needs to be true to work
// start parameters
var m = 1U << (int)BitsRequired(data.Max());
var length = f(m);
uint oldLength;
do
{
oldLength = length;
m /= 2;
length = f(m);
} while (length < oldLength && m > 1);
// now best between length and oldLength, binary search
// todo - search
Trace.Assert(m > 0);
var left = m;
var right = 2 * m;
var mid = 0U;
var a = f(left);
while (left <= right)
{
mid = (left + right) / 2;
var c = f(mid);
if (c < a)
{
left = mid + 1;
}
else
{
right = mid - 1;
}
}
if (mid == 1)
{
mid = 2;
}
// check mid, mid+1, mid-1
uint best;
if (f(mid) < f(mid + 1))
{
if (f(mid - 1) < f(mid))
{
best = mid - 1;
}
else
{
best = mid;
}
}
else
{
best = mid + 1;
}
if (Options.HasFlag(OptionFlags.DumpDebug))
{
WriteLine($"Golomb opt {mid} {f(mid-2)} {f(mid-1)} {f(mid)} {f(mid+1)} {f(mid+2)} {f(mid+3)}");
}
return(best);
}
/// <summary>
/// Write the header for the compression algorithm
/// </summary>
/// <param name="bitstream"></param>
/// <param name="data"></param>
/// <param name="headerFlags">Flags telling what to put in the header. Useful when embedding in other streams.</param>
/// <returns></returns>
public override void WriteHeader(Bitstream bitstream, Datastream data, Header.HeaderFlags headerFlags)
{
// erase data streams
decisions.Clear();
decisionRuns.Clear();
literals.Clear();
distances.Clear();
lengths.Clear();
tokens.Clear();
// fill in all the data streams
uint actualMinLength, actualMaxDistance;
ComputeStreams(data, out actualMinLength, out actualMaxDistance);
// due to the vagaries of this format, we write the entire file in the header call,
// and unfortunately ignore the encode symbol and footer sections
// dump info to help analyze
if (Options.HasFlag(OptionFlags.DumpDebug))
{
WriteLine("LZCL compress:");
WriteLine($" Data length {data.Count} ");
}
if (Options.HasFlag(OptionFlags.ShowTallies))
{
// some info to help make analyze and make decisions
Write("Length tally: ");
Tally(lengths);
WriteLine();
Write("Distance tally: ");
Tally(distances);
WriteLine();
}
// get compressed streams so we can decide what to output
var decisionChoice = GetBestCompressor("decisions", decisions);
var decisionRunsChoice = GetBestCompressor("decision runs", decisionRuns);
var literalsChoice = GetBestCompressor("literals", literals);
var tokensChoice = GetBestCompressor("tokens", tokens);
var distancesChoice = GetBestCompressor("distances", distances);
var lengthsChoice = GetBestCompressor("lengths", lengths);
// write header values
Header.WriteUniversalHeader(bitstream, data, headerFlags);
// save max distance occurring, used to encode tokens, very useful to users to know window needed size
UniversalCodec.Lomont.EncodeLomont1(bitstream, actualMaxDistance, 10, 0);
UniversalCodec.Lomont.EncodeLomont1(bitstream, actualMinLength, 2, 0);
if (Options.HasFlag(OptionFlags.DumpDebug))
{
WriteLine($"actual min length {actualMinLength}");
}
if (Options.HasFlag(OptionFlags.DumpDebug))
{
WriteLine($"Max distance {actualMaxDistance}");
}
if (decisionChoice.Item2.Length < decisionRunsChoice.Item2.Length)
{
// denote choice
bitstream.Write(0);
// save item
WriteItem(bitstream, decisionChoice);
if (Options.HasFlag(OptionFlags.DumpDebug))
{
WriteLine("Decisions smaller than decision runs");
}
StatRecorder.AddStat($"codec used: decisions {decisionChoice.Item1.Name}", 1);
}
else
{
// denote choice
bitstream.Write(1);
// save initial value
bitstream.Write(decisions[0]);
// save item
WriteItem(bitstream, decisionRunsChoice);
if (Options.HasFlag(OptionFlags.DumpDebug))
{
WriteLine("Decisions runs smaller than decisions");
}
StatRecorder.AddStat($"codec used: decision runs {decisionRunsChoice.Item1.Name}", 1);
}
// literals
WriteItem(bitstream, literalsChoice);
StatRecorder.AddStat($"codec used: literals {literalsChoice.Item1.Name}", 1);
// tokens or separate distance, length pairs
if (tokensChoice.Item2.Length < distancesChoice.Item2.Length + lengthsChoice.Item2.Length)
{
// denote choice
bitstream.Write(0);
// save item
WriteItem(bitstream, tokensChoice);
if (Options.HasFlag(OptionFlags.DumpDebug))
{
WriteLine("Tokens smaller than distance,length pairs");
}
StatRecorder.AddStat($"codec used: tokens {tokensChoice.Item1.Name}", 1);
}
else
{
// denote choice
bitstream.Write(1);
// save items
WriteItem(bitstream, distancesChoice);
WriteItem(bitstream, lengthsChoice);
if (Options.HasFlag(OptionFlags.DumpDebug))
{
WriteLine("Distance,length pairs smaller than tokens");
}
StatRecorder.AddStat($"codec used: distances {distancesChoice.Item1.Name}", 1);
StatRecorder.AddStat($"codec used: lengths {lengthsChoice.Item1.Name}", 1);
}
}
