public override void Encode(IntField image, Stream output)
{
// Predictive transform
image.ArgbTo4c();
image.PredictionEnTransformXor(Seer);
// Convert to three fields' runlengths
var fields = CodecUtil.FieldcodeRunlengthsEn(image, new RunLength01MaxSmartCodec(FieldcodeSymbols), this);
// Write size
DeltaTracker pos = new DeltaTracker();
output.WriteUInt32Optim((uint)image.Width);
output.WriteUInt32Optim((uint)image.Height);
SetCounter("bytes|size", pos.Next(output.Position));
// Write probs
ulong[] probs = CodecUtil.GetFreqsForAllSections(fields, FieldcodeSymbols + 1);
probs[0] = 6;
CodecUtil.SaveFreqsCrappy(output, probs);
SetCounter("bytes|probs", pos.Next(output.Position));
// Write fields
ArithmeticSectionsCodec ac = new ArithmeticSectionsCodec(probs, 6, output);
for (int i = 0; i < fields.Count; i++)
{
ac.WriteSection(fields[i]);
SetCounter("bytes|fields|" + (i + 1), pos.Next(output.Position));
}
ac.Encode();
SetCounter("bytes|arith-err", pos.Next(output.Position));
}
public override void Encode(IntField image, Stream output)
{
image.ArgbTo4c();
IntField orig = image.Clone();
//image.PredictionEnTransformXor(new HorzVertForeseer());
//IntField backgr = CodecUtil.BackgroundFilterThin(image, 2, 2);
IntField backgr = CodecUtil.BackgroundFilterSmall(image, 50);
AddImageGrayscale(image, "orig");
AddImageGrayscale(backgr, "backgr");
for (int p = 0; p < image.Data.Length; p++)
{
image.Data[p] ^= backgr.Data[p];
}
AddImageGrayscale(image, "foregr");
for (int i = 1; i <= 3; i++)
{
IntField field = image.Clone();
field.Conditional(pix => pix == i);
int[] syms = CodecUtil.LzwLinesEn(field, 4, 1);
AddImageGrayscale(field, "field{0}-{1}syms-max{2}".Fmt(i, syms.Length, syms.Max()));
}
}
public override void Encode(IntField image, Stream output)
{
// Predictive transform
image.ArgbTo4c();
image.PredictionEnTransformXor(Seer);
// Convert to three fields' runlengths
var fields = CodecUtil.FieldcodeRunlengthsEn2(image, RLE, this);
SetCounter("rle|longer", (RLE as RunLength01LongShortCodec).Counter_Longers);
SetCounter("rle|muchlonger", (RLE as RunLength01LongShortCodec).Counter_MuchLongers);
// Write size
DeltaTracker pos = new DeltaTracker();
output.WriteUInt32Optim((uint)image.Width);
output.WriteUInt32Optim((uint)image.Height);
SetCounter("bytes|size", pos.Next(output.Position));
// Write probs
ulong[] probs = CodecUtil.CountValues(fields, RLE.MaxSymbol);
CodecUtil.SaveFreqs(output, probs, TimwiCecCompressor.runLProbsProbs, "");
SetCounter("bytes|probs", pos.Next(output.Position));
// Write fields
ArithmeticWriter aw = new ArithmeticWriter(output, probs);
output.WriteUInt32Optim((uint)fields.Length);
foreach (var sym in fields)
{
aw.WriteSymbol(sym);
}
aw.Flush();
SetCounter("bytes|fields", pos.Next(output.Position));
}
public override void Encode(IntField image, Stream output)
{
/// Split background from foreground. Anything with low "rectangularity" goes into foreground
/// - measure the number of "turns" (changes from horz to vert edge) against area.
/// Background:
/// - Predictive xform with small radius
/// - Modified runlengths
/// - Context-aware arithmetic encoding
/// Foreground:
/// - Order colors by how common they are, call them c1,c2,c3
/// - Flatten all colors into 0 and 1
/// - Find all rects covering all c2-pixels without covering any c1-pixels.
/// - Find all rects covering all c3-pixels without covering any c1 or c2 pixels.
/// - Predictive xform on 0/1 image stopping at vertical letter boundaries. If larger image not in dict, use a smaller one and add all sizes to dict.
/// - Modified runlengths on the 0/1 xformed
/// - Context-aware arithmetic encoding
/// - Rects: optimize by changing 1 pixel rects to pixels; then feed optim ints through arithmetic
/// Modified runlengths:
/// - As before, encode only runs of 0's.
/// - Every run longer than thresh encoded as thresh + a number in a separate optim stream
/// - Optim stream is fed through arithmetic
///
/// Alternative color encoding: a stream of 1's, 2's, 3's, fed through runlength
image.ArgbTo4c();
AddImageGrayscale(image, 0, 3, "00-original");
IntField backgr, foregr;
backgroundSplit(image, out backgr, out foregr);
// Backgr
backgr.PredictionEnTransformXor(new FixedSizeForeseer(5, 3, 2, new HorzVertForeseer()));
AddImageGrayscale(backgr, 0, 3, "20-bkg-xformed");
saveColors(foregr);
// save as much as possible using rects, the rest just dumping as-is
// Foregr
IntField foremap = foregr.Clone();
foremap.Map(pix => pix == 0 ? 0 : 1);
AddImageGrayscale(foremap, 0, 1, "30-fore-map");
foremap.PredictionEnTransformXor(new FixedSizeForeseer(7, 7, 3, new HorzVertForeseer()));
AddImageGrayscale(foremap, 0, 1, "31-fore-map-xformed");
saveForeground(foremap.Data);
}
public override void Encode(IntField image, Stream output)
{
image.ArgbTo4c();
AddImageGrayscale(image, "res0");
List <IntField> scales = new List <IntField>();
scales.Add(image);
while (scales.Last().Width > 100 && scales.Last().Height > 100)
{
scales.Add(scales.Last().HalfResHighestCount());
}
for (int i = 1; i < scales.Count; i++)
{
IntField predicted = new IntField(scales[i - 1].Width, scales[i - 1].Height);
IntField shrunk = scales[i];
AddImageGrayscale(shrunk, "res" + i);
for (int y = 0; y < predicted.Height; y++)
{
for (int x = 0; x < predicted.Width; x++)
{
int xm2 = x & 1;
int ym2 = y & 1;
int xd2 = x >> 1;
int yd2 = y >> 1;
if (xm2 == 0 && ym2 == 0) // original
{
predicted[x, y] = shrunk[xd2, yd2];
}
else if (ym2 == 0) // between horizontal pixels
{
predicted[x, y] = shrunk[xd2, yd2];
}
else
{
predicted[x, y] = shrunk[xd2, yd2];
}
}
}
//AddImageGrayscale(predicted, "pred"+i);
for (int p = 0; p < predicted.Data.Length; p++)
{
predicted.Data[p] ^= scales[i - 1].Data[p];
}
//AddImageGrayscale(predicted, "diff"+i);
}
}
public override void Encode(IntField image, Stream output)
{
image.ArgbTo4c();
uint[] pixels = image.Data.Select(px => (uint)px).ToArray();
var newPixels = new uint[pixels.Length];
// XOR transform
for (int i = 0; i < pixels.Length; i++)
{
int x = i % image.Width;
uint p = pixels[i];
if (x > 0 && i >= image.Width)
{
if (pixels[i - 1] == pixels[i - image.Width - 1])
{
p = p ^ pixels[i - image.Width];
}
else
{
p = p ^ pixels[i - 1];
}
}
else if (x > 0)
{
p = p ^ pixels[i - 1];
}
else if (i >= image.Width)
{
p = p ^ pixels[i - image.Width];
}
newPixels[i] = p;
}
for (int i = 0; i < newPixels.Length; i++)
{
image.Data[i] = (int)newPixels[i];
}
AddImageGrayscale(image, "xformed");
base.Encode(image, output);
}
public override void Encode(IntField image, Stream output)
{
// Predictive transform
image.ArgbTo4c();
image.PredictionEnTransformXor(Seer);
// Convert to three fields' runlengths
for (int i = 1; i <= 3; i++)
{
IntField temp = image.Clone();
temp.Map(x => x == i ? 1 : 0);
AddImageGrayscale(temp, 0, 1, "field" + i);
var runs = new RunLength01SplitCodec().EncodeSplit(temp.Data);
SetCounter("runs|field{0}|0s".Fmt(i), runs.Item1.Count);
SetCounter("runs|field{0}|1s".Fmt(i), runs.Item2.Count);
AddIntDump("field{0}-0s".Fmt(i), runs.Item1);
AddIntDump("field{0}-1s".Fmt(i), runs.Item2);
}
}
public override void Encode(IntField image, Stream output)
{
image.ArgbTo4c();
switch (_mode)
{
case 0:
SetCounter("mispredicts-h", image.PredictionEnTransformXor(SeerH));
AddImageGrayscale(image, "xformed-h");
break;
case 1:
image.Transpose();
SetCounter("mispredicts-v", image.PredictionEnTransformXor(SeerV));
image.Transpose();
AddImageGrayscale(image, "xformed-v");
break;
case 2:
SetCounter("mispredicts-h", image.PredictionEnTransformXor(SeerH));
AddImageGrayscale(image, "xformed-h");
image.Transpose();
SetCounter("mispredicts-v", image.PredictionEnTransformXor(SeerV));
image.Transpose();
AddImageGrayscale(image, "xformed-v");
break;
case 3:
image.Transpose();
SetCounter("mispredicts-v", image.PredictionEnTransformXor(SeerV));
image.Transpose();
AddImageGrayscale(image, "xformed-v");
SetCounter("mispredicts-h", image.PredictionEnTransformXor(SeerH));
AddImageGrayscale(image, "xformed-h");
break;
default:
throw new Exception();
}
base.Encode(image, output);
}
public override void Encode(IntField image, Stream output)
{
image.ArgbTo4c();
image.PredictionEnTransformXor(Seer);
IntField[] fields = new IntField[4];
IntField[] cutmaps = new IntField[4];
List <Rectangle>[] areases = new List <Rectangle> [4];
for (int i = 1; i <= 3; i++)
{
fields[i] = image.Clone();
fields[i].Map(x => x == i ? 1 : 0);
cutmaps[i] = fields[i].ReduceKeepingPixels(ReduceBlocksize);
areases[i] = new List <Rectangle>();
IntField remains = fields[i].Clone();
while (true)
{
List <Rectangle> rects = CodecUtil.FindAllRects(cutmaps[i]);
if (rects.Count == 0 || rects[0].Width * rects[0].Height <= 1)
{
break;
}
Rectangle r = new Rectangle(rects[0].Left * ReduceBlocksize, rects[0].Top * ReduceBlocksize,
rects[0].Width * ReduceBlocksize, rects[0].Height * ReduceBlocksize);
r = CodecUtil.ShrinkRectangle(remains, r);
areases[i].Add(r);
cutmaps[i].ShadeRect(rects[0].Left, rects[0].Top, rects[0].Width, rects[0].Height, 0, 0);
remains.ShadeRect(r.Left, r.Top, r.Width, r.Height, 0, 0);
}
SetCounter("areas|" + i, areases[i].Count);
IntField vis = fields[i].Clone();
vis.ArgbFromField(0, 1);
foreach (var area in areases[i])
{
vis.ShadeRect(area.Left, area.Top, area.Width, area.Height, 0xFFFF7F7F, 0x007F0000);
}
for (int x = 0; x < cutmaps[i].Width; x++)
{
for (int y = 0; y < cutmaps[i].Height; y++)
{
if (cutmaps[i][x, y] > 0)
{
vis.ShadeRect(x * ReduceBlocksize, y * ReduceBlocksize, ReduceBlocksize, ReduceBlocksize,
0xFF7FFF7F, 0x00007F00);
}
}
}
AddImageArgb(vis, "vis" + i);
}
// now have: fields, covered in part by areas and in part by cutmap (only remaining cutmap left at this stage)
long pos = 0;
output.WriteInt32Optim(image.Width);
output.WriteInt32Optim(image.Height);
SetCounter("bytes|size", output.Position - pos);
pos = output.Position;
for (int i = 1; i <= 3; i++)
{
output.WriteInt32Optim(areases[i].Count);
}
SetCounter("bytes|areas|count", output.Position - pos);
pos = output.Position;
for (int i = 1; i <= 3; i++)
{
foreach (var area in areases[i])
{
output.WriteInt32Optim(area.Left);
output.WriteInt32Optim(area.Top);
}
SetCounter("bytes|areas|x,y|" + i, output.Position - pos);
pos = output.Position;
}
for (int i = 1; i <= 3; i++)
{
foreach (var area in areases[i])
{
output.WriteInt32Optim(area.Width);
output.WriteInt32Optim(area.Height);
}
SetCounter("bytes|areas|w,h|" + i, output.Position - pos);
pos = output.Position;
}
for (int i = 1; i <= 3; i++)
{
var pts = CodecUtil.GetPixelCoords(cutmaps[i], 1);
SetCounter("leftpixels|" + i, pts.Count);
output.WriteInt32Optim(pts.Count);
foreach (var pt in pts)
{
output.WriteInt32Optim(pt.X);
output.WriteInt32Optim(pt.Y);
}
}
RunLength01MaxSmartCodec runlen = new RunLength01MaxSmartCodec(FieldcodeSymbols);
List <int> data = new List <int>();
List <int> visdata = new List <int>();
for (int i = 1; i <= 3; i++)
{
foreach (var area in areases[i])
{
int[] aredata = fields[i].GetRectData(area);
data.AddRange(aredata);
visdata.AddRange(aredata.Select(val => unchecked ((int)0xFF000000) | ((i == 1 ? 0xF00000 : i == 2 ? 0xF08000 : 0xF00080) >> (2 - val * 2))));
fields[i].ShadeRect(area.Left, area.Top, area.Width, area.Height, 0, 0);
}
}
for (int i = 1; i <= 3; i++)
{
var pts = CodecUtil.GetPixelCoords(cutmaps[i], 1);
foreach (var pt in pts)
{
Rectangle rect = new Rectangle(pt.X * ReduceBlocksize, pt.Y * ReduceBlocksize, ReduceBlocksize, ReduceBlocksize);
int[] aredata = fields[i].GetRectData(rect);
data.AddRange(aredata);
visdata.AddRange(aredata.Select(val => unchecked ((int)0xFF000000) | ((i == 1 ? 0x00F000 : i == 2 ? 0x80F000 : 0x00F080) >> (2 - val * 2))));
}
}
int[] dataA = data.ToArray();
int[] symbols = runlen.Encode(dataA);
SetCounter("crux-pixels", data.Count);
SetCounter("crux-rle-symbols", symbols.Length);
int viw = (int)(Math.Sqrt(data.Count) * 1.3);
int vih = (int)Math.Ceiling((double)data.Count / viw);
IntField visual = new IntField(viw, vih);
Array.Copy(visdata.ToArray(), visual.Data, visdata.Count);
AddImageArgb(visual, "crux");
var probs = CodecUtil.CountValues(symbols);
output.WriteUInt32Optim((uint)probs.Length);
for (int p = 0; p < probs.Length; p++)
{
output.WriteUInt64Optim(probs[p]);
}
SetCounter("bytes|probs", output.Position - pos);
pos = output.Position;
ArithmeticWriter aw = new ArithmeticWriter(output, probs);
foreach (int sym in symbols)
{
aw.WriteSymbol(sym);
}
SetCounter("bytes|crux", output.Position - pos);
pos = output.Position;
aw.Flush();
// BETTER RLE - RUNS OF 1'S
// ARITH THE AREAS
// ARITH THE PROBS
// SHRINK GREENS
}
public override void Encode(IntField image, Stream output)
{
image.ArgbTo4c();
uint[] pixels = image.Data.Select(px => (uint)px).ToArray();
var newPixels = new uint[pixels.Length];
var predictor = new Efficient128bitHashTable <ulong[]>();
ulong context1 = 0, context2 = 0;
// Predictive transform
for (int i = 0; i < pixels.Length; i++)
{
int x = i % image.Width;
uint p = pixels[i];
bool fallback = false;
if (i < 7 * image.Width || x < 3)
{
fallback = true;
}
else
{
if (x == 3)
{
context1 = 0;
context2 = 0;
for (int yy = -7 * image.Width; yy <= 0; yy += image.Width)
{
for (int xx = -3; (xx <= 4) && (xx + yy < 0); xx++)
{
context1 = (context1 << 2) | ((context2 & 0xC000000000000000) >> 62);
context2 = (context2 << 2) | pixels[i + xx + yy];
}
}
}
else
{
context1 = ((context1 << 2) & 0x3FFF3FFF3FFF3C) | ((context2 & 0xC000000000000000) >> 62);
context2 = ((context2 << 2) & 0xFF3FFF3FFF3FFF3C) | (ulong)pixels[i - 1];
if (x + 4 < image.Width)
{
context1 |= ((ulong)pixels[i - 7 * image.Width + 4] << 38) | ((ulong)pixels[i - 6 * image.Width + 4] << 22) | ((ulong)pixels[i - 5 * image.Width + 4] << 6);
context2 |= ((ulong)pixels[i - 4 * image.Width + 4] << 54) | ((ulong)pixels[i - 3 * image.Width + 4] << 38) | ((ulong)pixels[i - 2 * image.Width + 4] << 22) | ((ulong)pixels[i - 1 * image.Width + 4] << 6);
}
}
ulong[] prev = predictor.Get(context1, context2);
if (prev == null)
{
prev = new ulong[4];
predictor.Add(context1, context2, prev);
fallback = true;
}
else
{
ulong p0 = prev[0], p1 = prev[1], p2 = prev[2], p3 = prev[3];
p = p ^ ((p0 >= p1 && p0 >= p2 && p0 >= p3) ? (byte)0 :
(p1 >= p2 && p1 >= p3) ? (byte)1 :
(p2 >= p3) ? (byte)2 : (byte)3);
}
prev[pixels[i]]++;
}
if (fallback)
{
// Fall back to old XOR transform
if (x > 0 && i >= image.Width)
{
if (pixels[i - 1] == pixels[i - image.Width - 1])
{
p = p ^ pixels[i - image.Width];
}
else
{
p = p ^ pixels[i - 1];
}
}
else if (x > 0)
{
p = p ^ pixels[i - 1];
}
else if (i >= image.Width)
{
p = p ^ pixels[i - image.Width];
}
}
newPixels[i] = p;
}
for (int i = 0; i < newPixels.Length; i++)
{
image.Data[i] = (int)newPixels[i];
}
AddImageGrayscale(image, "xformed");
base.Encode(image, output);
}
public override void Encode(IntField image, Stream output)
{
// Predictive transform
image.ArgbTo4c();
image.PredictionEnTransformXor(Seer);
int px = 0, py = 0;
List <int> dxs = new List <int>();
List <int> dys = new List <int>();
List <int> clr = new List <int>();
List <Point> jumps = new List <Point>();
IntField vis = new IntField(image.Width, image.Height);
int vis_ctr = 0;
while (true)
{
Point pt = FindNextPixel(image, px, py);
px += pt.X;
py += pt.Y;
int c = image.GetWrapped(px, py);
if (c == 0)
{
break;
}
if (Math.Abs(pt.X) > 5 || Math.Abs(pt.Y) > 5)
{
jumps.Add(pt);
}
else
{
dxs.Add(pt.X);
dys.Add(pt.Y);
}
clr.Add(c);
image.SetWrapped(px, py, 0);
if (vis_ctr % 1000 == 0)
{
AddImageGrayscale(image, "progress.{0:00000}".Fmt(vis_ctr));
}
vis.SetWrapped(px, py, ++vis_ctr);
}
SetCounter("jumps", jumps.Count);
AddIntDump("xs", dxs);
AddIntDump("ys", dys);
AddIntDump("cs", clr);
AddImageGrayscale(vis, "seq-global");
vis.Data = vis.Data.Select(val => val % 512).ToArray();
AddImageGrayscale(vis, "seq-local");
var xs = CodecUtil.InterleaveNegatives(dxs.ToArray());
var ys = CodecUtil.InterleaveNegatives(dxs.ToArray());
var cs = clr.ToArray();
List <ulong[]> xps = new List <ulong[]>();
List <ulong[]> yps = new List <ulong[]>();
List <ulong> xpts = new List <ulong>();
List <ulong> ypts = new List <ulong>();
for (int given = 0; given <= 10; given++)
{
if (given < 10)
{
xps.Add(CodecUtil.GetNextProbsGiven(xs, given));
yps.Add(CodecUtil.GetNextProbsGiven(ys, given));
}
else
{
xps.Add(CodecUtil.GetNextProbsGivenGreater(xs, given - 1));
yps.Add(CodecUtil.GetNextProbsGivenGreater(ys, given - 1));
}
AddIntDump("xp-{0}".Fmt(given), xps.Last().Select(var => (int)var));
AddIntDump("yp-{0}".Fmt(given), yps.Last().Select(var => (int)var));
xpts.Add(xps.Last().Aggregate((tot, val) => tot + val));
ypts.Add(yps.Last().Aggregate((tot, val) => tot + val));
}
List <ulong[]> cps = new List <ulong[]>();
cps.Add(new ulong[4] {
0, 1, 1, 1
});
for (int given = 1; given <= 3; given++)
{
cps.Add(CodecUtil.GetNextProbsGiven(cs, given));
AddIntDump("cp-{0}".Fmt(given), cps.Last().Select(var => (int)var));
}
ulong[] cpts = new ulong[4];
for (int i = 0; i < cps.Count; i++)
{
cpts[i] = cps[i].Aggregate((tot, val) => tot + val);
}
ArithmeticWriter aw = new ArithmeticWriter(output, null);
int prev;
//prev = 0;
//for (int i = 0; i < cs.Length; i++)
//{
// aw.Probs = cps[prev];
// aw.TotalProb = cpts[prev];
// aw.WriteSymbol(cs[i]);
// prev = cs[i];
//}
//aw.Flush();
// For comparison: normal arithmetic 5846, shifting probs 3270
//ulong[] probs = CodecUtil.CountValues(cs);
//AddIntDump("cp-overall", probs.Select(val => (int)val));
//ArithmeticWriter aw = new ArithmeticWriter(output, probs);
//for (int i = 0; i < cs.Length; i++)
// aw.WriteSymbol(cs[i]);
//aw.Flush();
prev = 0;
for (int i = 0; i < xs.Length; i++)
{
if (prev > 10)
{
prev = 10;
}
aw.Probs = xps[prev];
aw.TotalProb = xpts[prev];
aw.WriteSymbol(xs[i]);
prev = xs[i];
}
aw.Flush();
//prev = 0;
//for (int i = 0; i < ys.Length; i++)
//{
// if (prev > 10) prev = 10;
// aw.Probs = yps[prev];
// aw.TotalProb = ypts[prev];
// aw.WriteSymbol(ys[i]);
// prev = ys[i];
//}
//aw.Flush();
}