public void TestNormalByteSequence()
{
// The encoding of bytes under a 0..255 context with equal probability should end up outputting those bytes unchanged
var ms = new MemoryStream();
var encoder = new ArithmeticCodingWriter(ms, new ArithmeticSymbolArrayContext(256));
for (int i = 0; i <= 255; i++)
{
encoder.WriteSymbol(i);
}
encoder.Finalize(false);
var result = ms.ToArray();
for (int i = 0; i <= 255; i++)
{
Assert.AreEqual(i, result[i]);
}
ms = new MemoryStream(result);
var decoder = new ArithmeticCodingReader(ms, new ArithmeticSymbolArrayContext(256));
for (int i = 0; i <= 255; i++)
{
Assert.AreEqual(i, decoder.ReadSymbol());
}
decoder.Finalize(false);
}
public void TestBasic1()
{
var freqs = new uint[] { 10, 30, 10 }; // Symbol 0 occurs 10x, symbol 1 occurs 30x, symbol 2 occurs 10x
var ms = new MemoryStream();
var context = new ArithmeticSymbolArrayContext(freqs);
var encoder = new ArithmeticCodingWriter(ms, context);
for (int i = 0; i < 10; i++)
{
encoder.WriteSymbol(1);
encoder.WriteSymbol(0);
encoder.WriteSymbol(1);
encoder.WriteSymbol(2);
encoder.WriteSymbol(1);
}
encoder.Finalize();
ms.WriteByte(47);
ms = new MemoryStream(ms.ToArray());
var decoder = new ArithmeticCodingReader(ms, context);
for (int i = 0; i < 10; i++)
{
Assert.AreEqual(1, decoder.ReadSymbol());
Assert.AreEqual(0, decoder.ReadSymbol());
Assert.AreEqual(1, decoder.ReadSymbol());
Assert.AreEqual(2, decoder.ReadSymbol());
Assert.AreEqual(1, decoder.ReadSymbol());
}
decoder.Finalize();
Assert.AreEqual(47, ms.ReadByte());
}
public void TestRandom()
{
var rnd = new Random(468);
for (int t = 0; t < 2000; t++)
{
var freqs = newArray(rnd.Next(1, 300), _ => (uint)rnd.Next(1, 500));
var symbols = newArray(rnd.Next(0, 1000), _ => (uint)rnd.Next(0, freqs.Length)).Select(e => (int)e).ToArray();
var ms = new MemoryStream();
var encoder = new ArithmeticCodingWriter(ms, freqs);
for (int i = 0; i < symbols.Length; i++)
{
encoder.WriteSymbol(symbols[i]);
}
encoder.Finalize();
var expectedEnding = ms.Position;
ms.WriteByte(47);
var bytes = ms.ToArray();
ms = new MemoryStream(bytes);
var decoder = new ArithmeticCodingReader(ms, freqs);
for (int i = 0; i < symbols.Length; i++)
{
Assert.AreEqual(symbols[i], decoder.ReadSymbol());
}
decoder.Finalize();
Assert.AreEqual(expectedEnding, ms.Position);
Assert.AreEqual(47, ms.ReadByte());
}
}
private static void testExtremeProbs(uint[] freqs, int[] symbols)
{
var ms = new MemoryStream();
var encoder = new ArithmeticCodingWriter(ms, freqs);
foreach (var symbol in symbols)
{
encoder.WriteSymbol(symbol);
}
encoder.Finalize();
var expectedEnding = ms.Position;
ms.WriteByte(47);
var bytes = ms.ToArray();
ms = new MemoryStream(bytes);
var decoder = new ArithmeticCodingReader(ms, freqs);
for (int i = 0; i < symbols.Length; i++)
{
Assert.AreEqual(symbols[i], decoder.ReadSymbol());
}
decoder.Finalize();
Assert.AreEqual(expectedEnding, ms.Position);
Assert.AreEqual(47, ms.ReadByte());
}
private void testBasic(int length)
{
var freqs = newArray(256, v => 256 - (uint)v);
var ms = new MemoryStream();
var encoder = new ArithmeticCodingWriter(ms, freqs);
for (int i = 0; i < length; i++)
{
encoder.WriteSymbol(i % 256);
}
encoder.Finalize(false);
var expectedEnding = ms.Position;
ms.Write(new byte[32], 0, 32);
var bytes = ms.ToArray();
ms = new MemoryStream(bytes);
var decoder = new ArithmeticCodingReader(ms, freqs);
for (int i = 0; i < length; i++)
{
var sym = decoder.ReadSymbol();
Assert.AreEqual(i % 256, sym);
}
decoder.Finalize(false);
Assert.AreEqual(expectedEnding, ms.Position);
}
public void TestSingleSymbol()
{
var freqs = new uint[] { 1 };
var ms = new MemoryStream();
var encoder = new ArithmeticCodingWriter(ms, freqs);
for (int i = 0; i < 100; i++)
{
encoder.WriteSymbol(0);
}
encoder.Finalize(false);
Assert.AreEqual(5, ms.Position);
ms = new MemoryStream(ms.ToArray());
var decoder = new ArithmeticCodingReader(ms, freqs);
for (int i = 0; i < 100; i++)
{
Assert.AreEqual(0, decoder.ReadSymbol());
}
decoder.Finalize(false);
Assert.AreEqual(5, ms.Position);
}
public void TestExtremeProbabilities1()
{
// This test encodes and decodes a sequence of N 1's, followed by a single 0, where the frequency of 1 is extremely high and the frequency of 0 is 1 (minimal).
// For very large frequencies, the encoder overflows after a certain number of 1's.
// This test verifies correct operation for all sequences of length 1..1000, then goes up in bigger increments until a sequence of ~10 million 1's.
// A 100 billion long sequence has been tested manually but takes far too long for a unit test (encoded: FF FF FF FF FF FD 21 DB A1 79 + sync padding)
// Maximum frequency vs first failure at 1's count:
// 0xFFFF_FFFE: 2
// 0xFFFF_FFF0: 16
// 0xFFFF_FF00: 256
// 0xFFFF_F000: 4096
// 0xFFFF_0000: 65536
// 0xF000_0000: 268,435,455
// 0x8000_0001: 2,147,483,647
// 0x8000_0000: correct to at least 2.2 billion
// 0x7FFF_FFFF: correct to at least 100 billion
var freqs = new[] { 1u, ArithmeticSymbolContext.MaxTotal - 1 };
int count = 0;
while (true)
{
var ms = new MemoryStream();
var encoder = new ArithmeticCodingWriter(ms, freqs);
for (int i = 0; i < count; i++)
{
encoder.WriteSymbol(1);
}
encoder.WriteSymbol(0);
encoder.Finalize();
var expectedEnding = ms.Position;
ms.WriteByte(47);
var bytes = ms.ToArray();
ms = new MemoryStream(bytes);
var decoder = new ArithmeticCodingReader(ms, freqs);
for (int i = 0; i < count; i++)
{
Assert.AreEqual(1, decoder.ReadSymbol());
}
Assert.AreEqual(0, decoder.ReadSymbol());
decoder.Finalize();
Assert.AreEqual(expectedEnding, ms.Position);
Assert.AreEqual(47, ms.ReadByte());
if (count < 1000)
{
count++;
}
else if (count < 10_000)
{
count += 997;
}
else if (count < 100_000)
{
count = 9_999_991;
}
else
{
break;
}
}
}
public void TestAdvanced()
{
_rnd = new Random(12345);
int max = 1000;
var symbols = Enumerable.Range(1, 100_000).Select(_ => _rnd.Next(0, max)).ToArray();
var mainContext = new ArithmeticSymbolArrayContext(max, _ => 1);
var secondaryContext = new ArithmeticSymbolArrayContext(new uint[] { 3, 2, 1 });
var ms = new MemoryStream();
var encoder = new ArithmeticCodingWriter(ms, mainContext);
writeInt(ms, 12345);
for (int i = 0; i < symbols.Length; i++)
{
encoder.WriteSymbol(symbols[i]);
mainContext.IncrementSymbolFrequency(symbols[i]);
encoder.SetContext(mainContext);
if (i % 1000 == 999)
{
encoder.SetContext(secondaryContext);
encoder.WriteSymbol(0);
encoder.WriteSymbol(1);
encoder.WriteSymbol(0);
encoder.WriteSymbol(1);
encoder.WriteSymbol(0);
encoder.WriteSymbol(2);
encoder.SetContext(mainContext);
}
}
encoder.Finalize(false);
writeInt(ms, -54321); // to verify that the stream ends where we think it ends
var encoded = ms.ToArray();
ms = new MemoryStream(encoded);
mainContext = new ArithmeticSymbolArrayContext(max, _ => 1); // reset frequencies
Assert.AreEqual(12345, readInt(ms));
var decoder = new ArithmeticCodingReader(ms, mainContext);
for (int i = 0; i < symbols.Length; i++)
{
var sym = decoder.ReadSymbol();
Assert.AreEqual(symbols[i], sym);
mainContext.IncrementSymbolFrequency(sym);
decoder.SetContext(mainContext);
if (i % 1000 == 999)
{
decoder.SetContext(secondaryContext);
Assert.AreEqual(0, decoder.ReadSymbol());
Assert.AreEqual(1, decoder.ReadSymbol());
Assert.AreEqual(0, decoder.ReadSymbol());
Assert.AreEqual(1, decoder.ReadSymbol());
Assert.AreEqual(0, decoder.ReadSymbol());
Assert.AreEqual(2, decoder.ReadSymbol());
decoder.SetContext(mainContext);
}
}
decoder.Finalize(false);
Assert.AreEqual(-54321, readInt(ms));
}
public override void Encode(IntField image, Stream output)
{
int blocksX = (image.Width + _blocksizeX - 1) / _blocksizeX;
int blocksY = (image.Height + _blocksizeY - 1) / _blocksizeY;
int blocks = blocksX * blocksY;
int[] kinds = new int[blocks];
ulong[] blockHigh = new ulong[blocks];
ulong[] blockLow = new ulong[blocks];
var pixels = image.Data.Select(p => (uint)p).ToArray();
for (int i = 0; i < pixels.Length; i++)
{
int x = i % image.Width;
uint p = pixels[i];
if (p != 0)
{
int blockIndex = ((i / image.Width) / _blocksizeY) * blocksX + x / _blocksizeX;
kinds[blockIndex] |= 1 << (int)(p - 1);
int indexInBlock = ((i / image.Width) % _blocksizeY) * _blocksizeX + x % _blocksizeX;
if (indexInBlock < 32)
{
blockLow[blockIndex] |= (ulong)p << (2 * indexInBlock);
}
else
{
blockHigh[blockIndex] |= (ulong)p << (2 * (indexInBlock - 32));
}
}
}
// Start writing to the file
long pos = 0;
output.WriteUInt32Optim((uint)image.Width);
output.WriteUInt32Optim((uint)image.Height);
SetCounter("bytes|size", output.Position - pos);
pos = output.Position;
// Some slight optimisations
for (int i = 0; i < blocks; i++)
{
var k = kinds[i];
// If a kind-5 block is sandwiched between two kind-7s, it is worth setting to 7 to improve the run-length encoding
if (k == 5 && i > 0 && i < blocks - 1 && kinds[i - 1] == 7 && kinds[i + 1] == 7)
{
k = kinds[i] = 7;
}
// Kinds 3 and 6 are the rarest. It is more common for a block to have its top or bottom half all black.
// Therefore, redefine 3 and 6 to mean top and bottom half all black.
// A block that has kind 5 should also be changed to 3 or 6 if it has one half all black, because
// having kind 5 reduces the block's entropy by 1/3, but having one half all black by 1/2.
if (k == 3 || k > 4)
{
kinds[i] = blockLow[i] == 0 ? 3 : blockHigh[i] == 0 ? 6 : k == 5 ? 5 : 7;
}
}
// Encode the kinds as three planes of optim-encoded run lengths
long pos2 = 0;
var ms = new MemoryStream();
bool curRunData = false;
uint curRunLength = 0;
for (int c = 0; c < 3; c++)
{
for (int i = 0; i < blocks; i++)
{
if (curRunData ^ ((kinds[i] & (1 << c)) != 0))
{
ms.WriteUInt32Optim(curRunLength);
curRunData = !curRunData;
curRunLength = 1;
}
else
{
curRunLength++;
}
}
SetCounter("kinds-raw|" + c, ms.Position - pos2);
pos2 = ms.Position;
}
ms.WriteUInt32Optim(curRunLength);
ms.Close();
var kindsArr = ms.ToArray();
SetCounter("kinds-raw|error", kindsArr.Length - pos2);
// Save the frequencies of each byte in the Optim-encoded run lengths
ulong[] kFreq = new ulong[256];
for (int i = 0; i < kindsArr.Length; i++)
{
kFreq[kindsArr[i]]++;
}
CodecUtil.SaveFreqs(output, kFreq, kindProbsProbs, "kindProbsProbs");
SetCounter("bytes|kinds|probs", output.Position - pos);
pos = output.Position;
// Save the run lengths themselves
MemoryStream kMaster = new MemoryStream();
var kAcw = new ArithmeticCodingWriter(kMaster, kFreq);
foreach (var symb in kindsArr)
{
kAcw.WriteSymbol(symb);
}
kAcw.Close(false);
var kArr = kMaster.ToArray();
output.WriteUInt32Optim((uint)kindsArr.Length);
output.WriteUInt32Optim((uint)kArr.Length);
output.Write(kArr, 0, kArr.Length);
//Console.WriteLine("krl " + kArr.Length);
SetCounter("bytes|kinds|data", output.Position - pos);
pos = output.Position;
/*
* Bitmap bXor = PixelsToBitmap(newPixels, bw, bh);
* bXor.Save(target + "-xor.png");
* Bitmap bBlocks = KindsToBitmap(kinds, blocksizeX, blocksizeY, bw, bh);
* Graphics g = Graphics.FromImage(bXor);
* GraphicsUtil.DrawImageAlpha(g, bBlocks, new Rectangle(0, 0, bw, bh), 0.5f);
* bXor.Save(target + "-xor-blocks.png");
* /**/
// Create the sequence of symbols that represents the run lengths for the actual pixels
ms = new MemoryStream();
int j = 0;
ulong curLength = 0;
var kind = kinds[0];
while (j < 3 * pixels.Length)
{
var c = j / pixels.Length;
var i = j % pixels.Length;
var x = i % image.Width;
if (x % _blocksizeX == 0)
{
kind = kinds[((i / image.Width) / _blocksizeY) * blocksX + x / _blocksizeX];
}
j++;
if (kind == 3)
{
if (((i / image.Width) % _blocksizeY) * _blocksizeX + x % _blocksizeX < 32)
{
continue;
}
kind = 7;
}
else if (kind == 6)
{
if (((i / image.Width) % _blocksizeY) * _blocksizeX + x % _blocksizeX > 31)
{
continue;
}
kind = 7;
}
else if (((kind % 8) & (1 << c)) == 0)
{
continue;
}
if (pixels[i] == c + 1)
{
ms.WriteUInt64Optim(curLength);
curLength = 0;
}
else if (pixels[i] == 0 || pixels[i] > c + 1)
{
curLength++;
}
}
if (curLength > 0)
{
ms.WriteUInt64Optim(curLength);
}
ms.Close();
var runLengthsArr = ms.ToArray();
// Save the frequencies of each byte in the Optim-encoded run lengths
ulong[] freq = new ulong[256];
for (int i = 0; i < runLengthsArr.Length; i++)
{
freq[runLengthsArr[i]]++;
}
CodecUtil.SaveFreqs(output, freq, runLProbsProbs, "runLProbsProbs");
SetCounter("bytes|runs|probs", output.Position - pos);
pos = output.Position;
// Save the run lengths themselves
MemoryStream master = new MemoryStream();
master.WriteUInt32Optim((uint)runLengthsArr.Length);
var acw = new ArithmeticCodingWriter(master, freq);
foreach (var symb in runLengthsArr)
{
acw.WriteSymbol(symb);
}
acw.Close(false);
var arr = master.ToArray();
output.Write(arr, 0, arr.Length);
//Console.WriteLine("rl " + arr.Length);
SetCounter("bytes|runs|data", output.Position - pos);
pos = output.Position;
output.Close();
}
