/// <summary>
/// USAGE WARNING: this modifies "freqs", and the /modified/ version is the one
/// that is actually saved. Make sure to use the modified version in arithmetic codec.
/// </summary>
public static void SaveFreqs(Stream saveTo, ulong[] freqs, ulong[] probsProbs, string id)
{
int count = freqs.Length;
MemoryStream master = new MemoryStream();
MemoryStream ms = new MemoryStream();
var aw = new ArithmeticWriter(ms, probsProbs);
for (int i = 0; i < count; i++)
{
aw.WriteSymbol(freqs[i] >= 31 ? 31 : (int)freqs[i]);
}
aw.Flush();
var arr = ms.ToArray();
master.WriteUInt32Optim((uint)arr.Length);
master.Write(arr, 0, arr.Length);
for (int i = 0; i < count; i++)
{
if (freqs[i] >= 31)
{
freqs[i] -= freqs[i] % 31;
master.WriteUInt64Optim(freqs[i] / 31 - 1);
}
}
master.Close();
arr = master.ToArray();
saveTo.Write(arr, 0, arr.Length);
//counters.IncSafe("freq " + id, (ulong) arr.Length);
//Console.Write("freq " + arr.Length + "; ");
}
public override void Encode(IntField image, Stream output)
{
// Predictive transform
image.ArgbTo4c();
image.PredictionEnTransformXor(Seer);
// Convert to three fields' runlengths
var fields = CodecUtil.FieldcodeRunlengthsEn2(image, RLE, this);
SetCounter("rle|longer", (RLE as RunLength01LongShortCodec).Counter_Longers);
SetCounter("rle|muchlonger", (RLE as RunLength01LongShortCodec).Counter_MuchLongers);
// Write size
DeltaTracker pos = new DeltaTracker();
output.WriteUInt32Optim((uint)image.Width);
output.WriteUInt32Optim((uint)image.Height);
SetCounter("bytes|size", pos.Next(output.Position));
// Write probs
ulong[] probs = CodecUtil.CountValues(fields, RLE.MaxSymbol);
CodecUtil.SaveFreqs(output, probs, TimwiCecCompressor.runLProbsProbs, "");
SetCounter("bytes|probs", pos.Next(output.Position));
// Write fields
ArithmeticWriter aw = new ArithmeticWriter(output, probs);
output.WriteUInt32Optim((uint)fields.Length);
foreach (var sym in fields)
{
aw.WriteSymbol(sym);
}
aw.Flush();
SetCounter("bytes|fields", pos.Next(output.Position));
}
public byte[] Encode(int[] data)
{
MemoryStream ms = new MemoryStream();
ArithmeticWriter aw = new ArithmeticWriter(ms, _probs);
foreach (var sym in data)
{
aw.WriteSymbol(sym);
}
aw.WriteSymbol(_symEnd);
aw.Flush();
return(ms.ToArray());
}
public void WriteSection(int[] data)
{
if (_arithReader != null)
{
throw new InvalidOperationException("Cannot mix reads and writes in ArithmeticSectionsCodec");
}
if (_arithWriter == null)
{
_memStream = new MemoryStream();
_arithWriter = new ArithmeticWriter(_memStream, _probs);
}
foreach (var sym in data)
{
_arithWriter.WriteSymbol(sym);
}
_arithWriter.WriteSymbol(_symEnd);
}
private byte[] mrleEncodeShort(int[] data, ulong[][] probs)
{
MemoryStream ms = new MemoryStream();
ArithmeticWriter aw = new ArithmeticWriter(ms, null);
ulong[] tots = new ulong[probs.Length];
for (int i = 0; i < tots.Length; i++)
{
tots[i] = (ulong)probs[i].Sum(val => (long)val);
}
int prev = 0;
for (int p = 0; p < data.Length; p++)
{
aw.Probs = probs[prev];
aw.TotalProb = tots[prev];
aw.WriteSymbol(data[p]);
prev = data[p];
}
aw.Flush();
return(ms.ToArray());
}
public ArithmeticSectionsCodec(ulong[] probabilities, ulong estSections, Stream sourceStream)
: this(probabilities, estSections)
{
_arithWriter = new ArithmeticWriter(sourceStream, _probs);
}
public override void Encode(IntField image, Stream output)
{
image.ArgbTo4c();
image.PredictionEnTransformXor(Seer);
IntField[] fields = new IntField[4];
IntField[] cutmaps = new IntField[4];
List <Rectangle>[] areases = new List <Rectangle> [4];
for (int i = 1; i <= 3; i++)
{
fields[i] = image.Clone();
fields[i].Map(x => x == i ? 1 : 0);
cutmaps[i] = fields[i].ReduceKeepingPixels(ReduceBlocksize);
areases[i] = new List <Rectangle>();
IntField remains = fields[i].Clone();
while (true)
{
List <Rectangle> rects = CodecUtil.FindAllRects(cutmaps[i]);
if (rects.Count == 0 || rects[0].Width * rects[0].Height <= 1)
{
break;
}
Rectangle r = new Rectangle(rects[0].Left * ReduceBlocksize, rects[0].Top * ReduceBlocksize,
rects[0].Width * ReduceBlocksize, rects[0].Height * ReduceBlocksize);
r = CodecUtil.ShrinkRectangle(remains, r);
areases[i].Add(r);
cutmaps[i].ShadeRect(rects[0].Left, rects[0].Top, rects[0].Width, rects[0].Height, 0, 0);
remains.ShadeRect(r.Left, r.Top, r.Width, r.Height, 0, 0);
}
SetCounter("areas|" + i, areases[i].Count);
IntField vis = fields[i].Clone();
vis.ArgbFromField(0, 1);
foreach (var area in areases[i])
{
vis.ShadeRect(area.Left, area.Top, area.Width, area.Height, 0xFFFF7F7F, 0x007F0000);
}
for (int x = 0; x < cutmaps[i].Width; x++)
{
for (int y = 0; y < cutmaps[i].Height; y++)
{
if (cutmaps[i][x, y] > 0)
{
vis.ShadeRect(x * ReduceBlocksize, y * ReduceBlocksize, ReduceBlocksize, ReduceBlocksize,
0xFF7FFF7F, 0x00007F00);
}
}
}
AddImageArgb(vis, "vis" + i);
}
// now have: fields, covered in part by areas and in part by cutmap (only remaining cutmap left at this stage)
long pos = 0;
output.WriteInt32Optim(image.Width);
output.WriteInt32Optim(image.Height);
SetCounter("bytes|size", output.Position - pos);
pos = output.Position;
for (int i = 1; i <= 3; i++)
{
output.WriteInt32Optim(areases[i].Count);
}
SetCounter("bytes|areas|count", output.Position - pos);
pos = output.Position;
for (int i = 1; i <= 3; i++)
{
foreach (var area in areases[i])
{
output.WriteInt32Optim(area.Left);
output.WriteInt32Optim(area.Top);
}
SetCounter("bytes|areas|x,y|" + i, output.Position - pos);
pos = output.Position;
}
for (int i = 1; i <= 3; i++)
{
foreach (var area in areases[i])
{
output.WriteInt32Optim(area.Width);
output.WriteInt32Optim(area.Height);
}
SetCounter("bytes|areas|w,h|" + i, output.Position - pos);
pos = output.Position;
}
for (int i = 1; i <= 3; i++)
{
var pts = CodecUtil.GetPixelCoords(cutmaps[i], 1);
SetCounter("leftpixels|" + i, pts.Count);
output.WriteInt32Optim(pts.Count);
foreach (var pt in pts)
{
output.WriteInt32Optim(pt.X);
output.WriteInt32Optim(pt.Y);
}
}
RunLength01MaxSmartCodec runlen = new RunLength01MaxSmartCodec(FieldcodeSymbols);
List <int> data = new List <int>();
List <int> visdata = new List <int>();
for (int i = 1; i <= 3; i++)
{
foreach (var area in areases[i])
{
int[] aredata = fields[i].GetRectData(area);
data.AddRange(aredata);
visdata.AddRange(aredata.Select(val => unchecked ((int)0xFF000000) | ((i == 1 ? 0xF00000 : i == 2 ? 0xF08000 : 0xF00080) >> (2 - val * 2))));
fields[i].ShadeRect(area.Left, area.Top, area.Width, area.Height, 0, 0);
}
}
for (int i = 1; i <= 3; i++)
{
var pts = CodecUtil.GetPixelCoords(cutmaps[i], 1);
foreach (var pt in pts)
{
Rectangle rect = new Rectangle(pt.X * ReduceBlocksize, pt.Y * ReduceBlocksize, ReduceBlocksize, ReduceBlocksize);
int[] aredata = fields[i].GetRectData(rect);
data.AddRange(aredata);
visdata.AddRange(aredata.Select(val => unchecked ((int)0xFF000000) | ((i == 1 ? 0x00F000 : i == 2 ? 0x80F000 : 0x00F080) >> (2 - val * 2))));
}
}
int[] dataA = data.ToArray();
int[] symbols = runlen.Encode(dataA);
SetCounter("crux-pixels", data.Count);
SetCounter("crux-rle-symbols", symbols.Length);
int viw = (int)(Math.Sqrt(data.Count) * 1.3);
int vih = (int)Math.Ceiling((double)data.Count / viw);
IntField visual = new IntField(viw, vih);
Array.Copy(visdata.ToArray(), visual.Data, visdata.Count);
AddImageArgb(visual, "crux");
var probs = CodecUtil.CountValues(symbols);
output.WriteUInt32Optim((uint)probs.Length);
for (int p = 0; p < probs.Length; p++)
{
output.WriteUInt64Optim(probs[p]);
}
SetCounter("bytes|probs", output.Position - pos);
pos = output.Position;
ArithmeticWriter aw = new ArithmeticWriter(output, probs);
foreach (int sym in symbols)
{
aw.WriteSymbol(sym);
}
SetCounter("bytes|crux", output.Position - pos);
pos = output.Position;
aw.Flush();
// BETTER RLE - RUNS OF 1'S
// ARITH THE AREAS
// ARITH THE PROBS
// SHRINK GREENS
}
public override void Encode(IntField image, Stream output)
{
// Predictive transform
image.ArgbTo4c();
image.PredictionEnTransformXor(Seer);
int px = 0, py = 0;
List <int> dxs = new List <int>();
List <int> dys = new List <int>();
List <int> clr = new List <int>();
List <Point> jumps = new List <Point>();
IntField vis = new IntField(image.Width, image.Height);
int vis_ctr = 0;
while (true)
{
Point pt = FindNextPixel(image, px, py);
px += pt.X;
py += pt.Y;
int c = image.GetWrapped(px, py);
if (c == 0)
{
break;
}
if (Math.Abs(pt.X) > 5 || Math.Abs(pt.Y) > 5)
{
jumps.Add(pt);
}
else
{
dxs.Add(pt.X);
dys.Add(pt.Y);
}
clr.Add(c);
image.SetWrapped(px, py, 0);
if (vis_ctr % 1000 == 0)
{
AddImageGrayscale(image, "progress.{0:00000}".Fmt(vis_ctr));
}
vis.SetWrapped(px, py, ++vis_ctr);
}
SetCounter("jumps", jumps.Count);
AddIntDump("xs", dxs);
AddIntDump("ys", dys);
AddIntDump("cs", clr);
AddImageGrayscale(vis, "seq-global");
vis.Data = vis.Data.Select(val => val % 512).ToArray();
AddImageGrayscale(vis, "seq-local");
var xs = CodecUtil.InterleaveNegatives(dxs.ToArray());
var ys = CodecUtil.InterleaveNegatives(dxs.ToArray());
var cs = clr.ToArray();
List <ulong[]> xps = new List <ulong[]>();
List <ulong[]> yps = new List <ulong[]>();
List <ulong> xpts = new List <ulong>();
List <ulong> ypts = new List <ulong>();
for (int given = 0; given <= 10; given++)
{
if (given < 10)
{
xps.Add(CodecUtil.GetNextProbsGiven(xs, given));
yps.Add(CodecUtil.GetNextProbsGiven(ys, given));
}
else
{
xps.Add(CodecUtil.GetNextProbsGivenGreater(xs, given - 1));
yps.Add(CodecUtil.GetNextProbsGivenGreater(ys, given - 1));
}
AddIntDump("xp-{0}".Fmt(given), xps.Last().Select(var => (int)var));
AddIntDump("yp-{0}".Fmt(given), yps.Last().Select(var => (int)var));
xpts.Add(xps.Last().Aggregate((tot, val) => tot + val));
ypts.Add(yps.Last().Aggregate((tot, val) => tot + val));
}
List <ulong[]> cps = new List <ulong[]>();
cps.Add(new ulong[4] {
0, 1, 1, 1
});
for (int given = 1; given <= 3; given++)
{
cps.Add(CodecUtil.GetNextProbsGiven(cs, given));
AddIntDump("cp-{0}".Fmt(given), cps.Last().Select(var => (int)var));
}
ulong[] cpts = new ulong[4];
for (int i = 0; i < cps.Count; i++)
{
cpts[i] = cps[i].Aggregate((tot, val) => tot + val);
}
ArithmeticWriter aw = new ArithmeticWriter(output, null);
int prev;
//prev = 0;
//for (int i = 0; i < cs.Length; i++)
//{
// aw.Probs = cps[prev];
// aw.TotalProb = cpts[prev];
// aw.WriteSymbol(cs[i]);
// prev = cs[i];
//}
//aw.Flush();
// For comparison: normal arithmetic 5846, shifting probs 3270
//ulong[] probs = CodecUtil.CountValues(cs);
//AddIntDump("cp-overall", probs.Select(val => (int)val));
//ArithmeticWriter aw = new ArithmeticWriter(output, probs);
//for (int i = 0; i < cs.Length; i++)
// aw.WriteSymbol(cs[i]);
//aw.Flush();
prev = 0;
for (int i = 0; i < xs.Length; i++)
{
if (prev > 10)
{
prev = 10;
}
aw.Probs = xps[prev];
aw.TotalProb = xpts[prev];
aw.WriteSymbol(xs[i]);
prev = xs[i];
}
aw.Flush();
//prev = 0;
//for (int i = 0; i < ys.Length; i++)
//{
// if (prev > 10) prev = 10;
// aw.Probs = yps[prev];
// aw.TotalProb = ypts[prev];
// aw.WriteSymbol(ys[i]);
// prev = ys[i];
//}
//aw.Flush();
}
