public override void Encode(IntField image, Stream output)
{
int blocksX = (image.Width + _blocksizeX - 1) / _blocksizeX;
int blocksY = (image.Height + _blocksizeY - 1) / _blocksizeY;
int blocks = blocksX * blocksY;
int[] kinds = new int[blocks];
ulong[] blockHigh = new ulong[blocks];
ulong[] blockLow = new ulong[blocks];
var pixels = image.Data.Select(p => (uint)p).ToArray();
for (int i = 0; i < pixels.Length; i++)
{
int x = i % image.Width;
uint p = pixels[i];
if (p != 0)
{
int blockIndex = ((i / image.Width) / _blocksizeY) * blocksX + x / _blocksizeX;
kinds[blockIndex] |= 1 << (int)(p - 1);
int indexInBlock = ((i / image.Width) % _blocksizeY) * _blocksizeX + x % _blocksizeX;
if (indexInBlock < 32)
{
blockLow[blockIndex] |= (ulong)p << (2 * indexInBlock);
}
else
{
blockHigh[blockIndex] |= (ulong)p << (2 * (indexInBlock - 32));
}
}
}
// Start writing to the file
long pos = 0;
output.WriteUInt32Optim((uint)image.Width);
output.WriteUInt32Optim((uint)image.Height);
SetCounter("bytes|size", output.Position - pos);
pos = output.Position;
// Some slight optimisations
for (int i = 0; i < blocks; i++)
{
var k = kinds[i];
// If a kind-5 block is sandwiched between two kind-7s, it is worth setting to 7 to improve the run-length encoding
if (k == 5 && i > 0 && i < blocks - 1 && kinds[i - 1] == 7 && kinds[i + 1] == 7)
{
k = kinds[i] = 7;
}
// Kinds 3 and 6 are the rarest. It is more common for a block to have its top or bottom half all black.
// Therefore, redefine 3 and 6 to mean top and bottom half all black.
// A block that has kind 5 should also be changed to 3 or 6 if it has one half all black, because
// having kind 5 reduces the block's entropy by 1/3, but having one half all black by 1/2.
if (k == 3 || k > 4)
{
kinds[i] = blockLow[i] == 0 ? 3 : blockHigh[i] == 0 ? 6 : k == 5 ? 5 : 7;
}
}
// Encode the kinds as three planes of optim-encoded run lengths
long pos2 = 0;
var ms = new MemoryStream();
bool curRunData = false;
uint curRunLength = 0;
for (int c = 0; c < 3; c++)
{
for (int i = 0; i < blocks; i++)
{
if (curRunData ^ ((kinds[i] & (1 << c)) != 0))
{
ms.WriteUInt32Optim(curRunLength);
curRunData = !curRunData;
curRunLength = 1;
}
else
{
curRunLength++;
}
}
SetCounter("kinds-raw|" + c, ms.Position - pos2);
pos2 = ms.Position;
}
ms.WriteUInt32Optim(curRunLength);
ms.Close();
var kindsArr = ms.ToArray();
SetCounter("kinds-raw|error", kindsArr.Length - pos2);
// Save the frequencies of each byte in the Optim-encoded run lengths
ulong[] kFreq = new ulong[256];
for (int i = 0; i < kindsArr.Length; i++)
{
kFreq[kindsArr[i]]++;
}
CodecUtil.SaveFreqs(output, kFreq, kindProbsProbs, "kindProbsProbs");
SetCounter("bytes|kinds|probs", output.Position - pos);
pos = output.Position;
// Save the run lengths themselves
MemoryStream kMaster = new MemoryStream();
var kAcw = new ArithmeticCodingWriter(kMaster, kFreq);
foreach (var symb in kindsArr)
{
kAcw.WriteSymbol(symb);
}
kAcw.Close(false);
var kArr = kMaster.ToArray();
output.WriteUInt32Optim((uint)kindsArr.Length);
output.WriteUInt32Optim((uint)kArr.Length);
output.Write(kArr, 0, kArr.Length);
//Console.WriteLine("krl " + kArr.Length);
SetCounter("bytes|kinds|data", output.Position - pos);
pos = output.Position;
/*
* Bitmap bXor = PixelsToBitmap(newPixels, bw, bh);
* bXor.Save(target + "-xor.png");
* Bitmap bBlocks = KindsToBitmap(kinds, blocksizeX, blocksizeY, bw, bh);
* Graphics g = Graphics.FromImage(bXor);
* GraphicsUtil.DrawImageAlpha(g, bBlocks, new Rectangle(0, 0, bw, bh), 0.5f);
* bXor.Save(target + "-xor-blocks.png");
* /**/
// Create the sequence of symbols that represents the run lengths for the actual pixels
ms = new MemoryStream();
int j = 0;
ulong curLength = 0;
var kind = kinds[0];
while (j < 3 * pixels.Length)
{
var c = j / pixels.Length;
var i = j % pixels.Length;
var x = i % image.Width;
if (x % _blocksizeX == 0)
{
kind = kinds[((i / image.Width) / _blocksizeY) * blocksX + x / _blocksizeX];
}
j++;
if (kind == 3)
{
if (((i / image.Width) % _blocksizeY) * _blocksizeX + x % _blocksizeX < 32)
{
continue;
}
kind = 7;
}
else if (kind == 6)
{
if (((i / image.Width) % _blocksizeY) * _blocksizeX + x % _blocksizeX > 31)
{
continue;
}
kind = 7;
}
else if (((kind % 8) & (1 << c)) == 0)
{
continue;
}
if (pixels[i] == c + 1)
{
ms.WriteUInt64Optim(curLength);
curLength = 0;
}
else if (pixels[i] == 0 || pixels[i] > c + 1)
{
curLength++;
}
}
if (curLength > 0)
{
ms.WriteUInt64Optim(curLength);
}
ms.Close();
var runLengthsArr = ms.ToArray();
// Save the frequencies of each byte in the Optim-encoded run lengths
ulong[] freq = new ulong[256];
for (int i = 0; i < runLengthsArr.Length; i++)
{
freq[runLengthsArr[i]]++;
}
CodecUtil.SaveFreqs(output, freq, runLProbsProbs, "runLProbsProbs");
SetCounter("bytes|runs|probs", output.Position - pos);
pos = output.Position;
// Save the run lengths themselves
MemoryStream master = new MemoryStream();
master.WriteUInt32Optim((uint)runLengthsArr.Length);
var acw = new ArithmeticCodingWriter(master, freq);
foreach (var symb in runLengthsArr)
{
acw.WriteSymbol(symb);
}
acw.Close(false);
var arr = master.ToArray();
output.Write(arr, 0, arr.Length);
//Console.WriteLine("rl " + arr.Length);
SetCounter("bytes|runs|data", output.Position - pos);
pos = output.Position;
output.Close();
}
public override void Encode(IntField image, Stream output)
{
image.ArgbTo4c();
image.PredictionEnTransformXor(Seer);
IntField[] fields = new IntField[4];
IntField[] cutmaps = new IntField[4];
List <Rectangle>[] areases = new List <Rectangle> [4];
for (int i = 1; i <= 3; i++)
{
fields[i] = image.Clone();
fields[i].Map(x => x == i ? 1 : 0);
cutmaps[i] = fields[i].ReduceKeepingPixels(ReduceBlocksize);
areases[i] = new List <Rectangle>();
IntField remains = fields[i].Clone();
while (true)
{
List <Rectangle> rects = CodecUtil.FindAllRects(cutmaps[i]);
if (rects.Count == 0 || rects[0].Width * rects[0].Height <= 1)
{
break;
}
Rectangle r = new Rectangle(rects[0].Left * ReduceBlocksize, rects[0].Top * ReduceBlocksize,
rects[0].Width * ReduceBlocksize, rects[0].Height * ReduceBlocksize);
r = CodecUtil.ShrinkRectangle(remains, r);
areases[i].Add(r);
cutmaps[i].ShadeRect(rects[0].Left, rects[0].Top, rects[0].Width, rects[0].Height, 0, 0);
remains.ShadeRect(r.Left, r.Top, r.Width, r.Height, 0, 0);
}
SetCounter("areas|" + i, areases[i].Count);
IntField vis = fields[i].Clone();
vis.ArgbFromField(0, 1);
foreach (var area in areases[i])
{
vis.ShadeRect(area.Left, area.Top, area.Width, area.Height, 0xFFFF7F7F, 0x007F0000);
}
for (int x = 0; x < cutmaps[i].Width; x++)
{
for (int y = 0; y < cutmaps[i].Height; y++)
{
if (cutmaps[i][x, y] > 0)
{
vis.ShadeRect(x * ReduceBlocksize, y * ReduceBlocksize, ReduceBlocksize, ReduceBlocksize,
0xFF7FFF7F, 0x00007F00);
}
}
}
AddImageArgb(vis, "vis" + i);
}
// now have: fields, covered in part by areas and in part by cutmap (only remaining cutmap left at this stage)
long pos = 0;
output.WriteInt32Optim(image.Width);
output.WriteInt32Optim(image.Height);
SetCounter("bytes|size", output.Position - pos);
pos = output.Position;
for (int i = 1; i <= 3; i++)
{
output.WriteInt32Optim(areases[i].Count);
}
SetCounter("bytes|areas|count", output.Position - pos);
pos = output.Position;
for (int i = 1; i <= 3; i++)
{
foreach (var area in areases[i])
{
output.WriteInt32Optim(area.Left);
output.WriteInt32Optim(area.Top);
}
SetCounter("bytes|areas|x,y|" + i, output.Position - pos);
pos = output.Position;
}
for (int i = 1; i <= 3; i++)
{
foreach (var area in areases[i])
{
output.WriteInt32Optim(area.Width);
output.WriteInt32Optim(area.Height);
}
SetCounter("bytes|areas|w,h|" + i, output.Position - pos);
pos = output.Position;
}
for (int i = 1; i <= 3; i++)
{
var pts = CodecUtil.GetPixelCoords(cutmaps[i], 1);
SetCounter("leftpixels|" + i, pts.Count);
output.WriteInt32Optim(pts.Count);
foreach (var pt in pts)
{
output.WriteInt32Optim(pt.X);
output.WriteInt32Optim(pt.Y);
}
}
RunLength01MaxSmartCodec runlen = new RunLength01MaxSmartCodec(FieldcodeSymbols);
List <int> data = new List <int>();
List <int> visdata = new List <int>();
for (int i = 1; i <= 3; i++)
{
foreach (var area in areases[i])
{
int[] aredata = fields[i].GetRectData(area);
data.AddRange(aredata);
visdata.AddRange(aredata.Select(val => unchecked ((int)0xFF000000) | ((i == 1 ? 0xF00000 : i == 2 ? 0xF08000 : 0xF00080) >> (2 - val * 2))));
fields[i].ShadeRect(area.Left, area.Top, area.Width, area.Height, 0, 0);
}
}
for (int i = 1; i <= 3; i++)
{
var pts = CodecUtil.GetPixelCoords(cutmaps[i], 1);
foreach (var pt in pts)
{
Rectangle rect = new Rectangle(pt.X * ReduceBlocksize, pt.Y * ReduceBlocksize, ReduceBlocksize, ReduceBlocksize);
int[] aredata = fields[i].GetRectData(rect);
data.AddRange(aredata);
visdata.AddRange(aredata.Select(val => unchecked ((int)0xFF000000) | ((i == 1 ? 0x00F000 : i == 2 ? 0x80F000 : 0x00F080) >> (2 - val * 2))));
}
}
int[] dataA = data.ToArray();
int[] symbols = runlen.Encode(dataA);
SetCounter("crux-pixels", data.Count);
SetCounter("crux-rle-symbols", symbols.Length);
int viw = (int)(Math.Sqrt(data.Count) * 1.3);
int vih = (int)Math.Ceiling((double)data.Count / viw);
IntField visual = new IntField(viw, vih);
Array.Copy(visdata.ToArray(), visual.Data, visdata.Count);
AddImageArgb(visual, "crux");
var probs = CodecUtil.CountValues(symbols);
output.WriteUInt32Optim((uint)probs.Length);
for (int p = 0; p < probs.Length; p++)
{
output.WriteUInt64Optim(probs[p]);
}
SetCounter("bytes|probs", output.Position - pos);
pos = output.Position;
ArithmeticWriter aw = new ArithmeticWriter(output, probs);
foreach (int sym in symbols)
{
aw.WriteSymbol(sym);
}
SetCounter("bytes|crux", output.Position - pos);
pos = output.Position;
aw.Flush();
// BETTER RLE - RUNS OF 1'S
// ARITH THE AREAS
// ARITH THE PROBS
// SHRINK GREENS
}
public override void Encode(IntField image, Stream output)
{
// Predictive transform
image.ArgbTo4c();
image.PredictionEnTransformXor(Seer);
int px = 0, py = 0;
List <int> dxs = new List <int>();
List <int> dys = new List <int>();
List <int> clr = new List <int>();
List <Point> jumps = new List <Point>();
IntField vis = new IntField(image.Width, image.Height);
int vis_ctr = 0;
while (true)
{
Point pt = FindNextPixel(image, px, py);
px += pt.X;
py += pt.Y;
int c = image.GetWrapped(px, py);
if (c == 0)
{
break;
}
if (Math.Abs(pt.X) > 5 || Math.Abs(pt.Y) > 5)
{
jumps.Add(pt);
}
else
{
dxs.Add(pt.X);
dys.Add(pt.Y);
}
clr.Add(c);
image.SetWrapped(px, py, 0);
if (vis_ctr % 1000 == 0)
{
AddImageGrayscale(image, "progress.{0:00000}".Fmt(vis_ctr));
}
vis.SetWrapped(px, py, ++vis_ctr);
}
SetCounter("jumps", jumps.Count);
AddIntDump("xs", dxs);
AddIntDump("ys", dys);
AddIntDump("cs", clr);
AddImageGrayscale(vis, "seq-global");
vis.Data = vis.Data.Select(val => val % 512).ToArray();
AddImageGrayscale(vis, "seq-local");
var xs = CodecUtil.InterleaveNegatives(dxs.ToArray());
var ys = CodecUtil.InterleaveNegatives(dxs.ToArray());
var cs = clr.ToArray();
List <ulong[]> xps = new List <ulong[]>();
List <ulong[]> yps = new List <ulong[]>();
List <ulong> xpts = new List <ulong>();
List <ulong> ypts = new List <ulong>();
for (int given = 0; given <= 10; given++)
{
if (given < 10)
{
xps.Add(CodecUtil.GetNextProbsGiven(xs, given));
yps.Add(CodecUtil.GetNextProbsGiven(ys, given));
}
else
{
xps.Add(CodecUtil.GetNextProbsGivenGreater(xs, given - 1));
yps.Add(CodecUtil.GetNextProbsGivenGreater(ys, given - 1));
}
AddIntDump("xp-{0}".Fmt(given), xps.Last().Select(var => (int)var));
AddIntDump("yp-{0}".Fmt(given), yps.Last().Select(var => (int)var));
xpts.Add(xps.Last().Aggregate((tot, val) => tot + val));
ypts.Add(yps.Last().Aggregate((tot, val) => tot + val));
}
List <ulong[]> cps = new List <ulong[]>();
cps.Add(new ulong[4] {
0, 1, 1, 1
});
for (int given = 1; given <= 3; given++)
{
cps.Add(CodecUtil.GetNextProbsGiven(cs, given));
AddIntDump("cp-{0}".Fmt(given), cps.Last().Select(var => (int)var));
}
ulong[] cpts = new ulong[4];
for (int i = 0; i < cps.Count; i++)
{
cpts[i] = cps[i].Aggregate((tot, val) => tot + val);
}
ArithmeticWriter aw = new ArithmeticWriter(output, null);
int prev;
//prev = 0;
//for (int i = 0; i < cs.Length; i++)
//{
// aw.Probs = cps[prev];
// aw.TotalProb = cpts[prev];
// aw.WriteSymbol(cs[i]);
// prev = cs[i];
//}
//aw.Flush();
// For comparison: normal arithmetic 5846, shifting probs 3270
//ulong[] probs = CodecUtil.CountValues(cs);
//AddIntDump("cp-overall", probs.Select(val => (int)val));
//ArithmeticWriter aw = new ArithmeticWriter(output, probs);
//for (int i = 0; i < cs.Length; i++)
// aw.WriteSymbol(cs[i]);
//aw.Flush();
prev = 0;
for (int i = 0; i < xs.Length; i++)
{
if (prev > 10)
{
prev = 10;
}
aw.Probs = xps[prev];
aw.TotalProb = xpts[prev];
aw.WriteSymbol(xs[i]);
prev = xs[i];
}
aw.Flush();
//prev = 0;
//for (int i = 0; i < ys.Length; i++)
//{
// if (prev > 10) prev = 10;
// aw.Probs = yps[prev];
// aw.TotalProb = ypts[prev];
// aw.WriteSymbol(ys[i]);
// prev = ys[i];
//}
//aw.Flush();
}