public override int Foresee(IntField image, int x, int y)
{
int l = x == 0 ? 0 : image[x - 1, y];
int t = y == 0 ? 0 : image[x, y - 1];
int tl = (x == 0 || y == 0) ? 0 : image[x - 1, y - 1];
if (l == tl && t == tl)
{
return(tl); // solid fill
}
else if (l == tl)
{
return(t); // vertical line
}
else if (t == tl)
{
return(l); // horizontal line
}
else if (t == l)
{
return(tl); // 50% diffusion fill
}
else
{
return(tl); // favour diagonals from top-left to bottom-right
}
}
public static int MostFrequent(IntField image, int ptX, int ptY, int lookbackW, int lookbackH)
{
Dictionary <int, int> counts = new Dictionary <int, int>();
for (int y = ptY - lookbackH + 1; y <= ptY; y++)
{
for (int x = ptX - lookbackW + 1; x <= ptX; x++)
{
if (y != ptY && x != ptX && y >= 0 && x >= 0)
{
int sym = image[x, y];
if (counts.ContainsKey(sym))
{
counts[sym]++;
}
else
{
counts.Add(sym, 1);
}
}
}
}
if (counts.Count == 0)
{
return(0);
}
int maxcount = counts.Values.Max();
return(counts.Where(kvp => kvp.Value == maxcount).First().Key);
}
public override int Foresee(IntField image, int x, int y)
{
pattern horzProbe = findPattern(image, x - _horzLookback, y - _horzThickness + 1, _horzLookback, _horzThickness);
pattern vertProbe = findPattern(image, x - _vertThickness + 1, y - _vertLookback, _vertThickness, _vertLookback);
if (horzProbe == pattern.Dirty && vertProbe == pattern.Dirty)
{
return(MostFrequentForeseer.MostFrequent(image, x, y, _freqThickness, _freqThickness));
}
bool haveH = horzProbe == pattern.CleanHorz || vertProbe == pattern.CleanHorz;
bool haveV = horzProbe == pattern.CleanVert || vertProbe == pattern.CleanVert;
if (haveH && !haveV)
{
return(MostFrequentForeseer.MostFrequent(image, x, y, _horzLookback, _horzThickness));
}
if (haveV && !haveH)
{
return(MostFrequentForeseer.MostFrequent(image, x, y, _vertThickness, _vertLookback));
}
if (horzProbe == pattern.Solid)
{
return(MostFrequentForeseer.MostFrequent(image, x, y, _horzLookback, _horzThickness));
}
else
{
return(MostFrequentForeseer.MostFrequent(image, x, y, _vertThickness, _vertLookback));
}
}
public override void Encode(IntField image, Stream output)
{
image.ArgbTo4c();
IntField orig = image.Clone();
//image.PredictionEnTransformXor(new HorzVertForeseer());
//IntField backgr = CodecUtil.BackgroundFilterThin(image, 2, 2);
IntField backgr = CodecUtil.BackgroundFilterSmall(image, 50);
AddImageGrayscale(image, "orig");
AddImageGrayscale(backgr, "backgr");
for (int p = 0; p < image.Data.Length; p++)
{
image.Data[p] ^= backgr.Data[p];
}
AddImageGrayscale(image, "foregr");
for (int i = 1; i <= 3; i++)
{
IntField field = image.Clone();
field.Conditional(pix => pix == i);
int[] syms = CodecUtil.LzwLinesEn(field, 4, 1);
AddImageGrayscale(field, "field{0}-{1}syms-max{2}".Fmt(i, syms.Length, syms.Max()));
}
}
public static int[] LzwLinesEn(IntField bitfield, int lineheight, int backadd)
{
int[] symbols = BitImageToLineSymbols(bitfield, lineheight);
LzwCodec lzw = new LzwCodec(1 << lineheight);
return(lzw.Encode(symbols));
}
public override void Encode(IntField image, Stream output)
{
// Predictive transform
image.ArgbTo4c();
image.PredictionEnTransformXor(Seer);
// Convert to three fields' runlengths
var fields = CodecUtil.FieldcodeRunlengthsEn(image, new RunLength01MaxSmartCodec(FieldcodeSymbols), this);
// Write size
DeltaTracker pos = new DeltaTracker();
output.WriteUInt32Optim((uint)image.Width);
output.WriteUInt32Optim((uint)image.Height);
SetCounter("bytes|size", pos.Next(output.Position));
// Write probs
ulong[] probs = CodecUtil.GetFreqsForAllSections(fields, FieldcodeSymbols + 1);
probs[0] = 6;
CodecUtil.SaveFreqsCrappy(output, probs);
SetCounter("bytes|probs", pos.Next(output.Position));
// Write fields
ArithmeticSectionsCodec ac = new ArithmeticSectionsCodec(probs, 6, output);
for (int i = 0; i < fields.Count; i++)
{
ac.WriteSection(fields[i]);
SetCounter("bytes|fields|" + (i + 1), pos.Next(output.Position));
}
ac.Encode();
SetCounter("bytes|arith-err", pos.Next(output.Position));
}
public void AddImageGrayscale(IntField image, int min, int max, string caption)
{
IntField temp = image.Clone();
temp.ArgbFromField(min, max);
Images.Add(new Tuple <string, IntField>(caption, temp));
}
public override void Encode(IntField image, Stream output)
{
// Predictive transform
image.ArgbTo4c();
image.PredictionEnTransformXor(Seer);
// Convert to three fields' runlengths
var fields = CodecUtil.FieldcodeRunlengthsEn2(image, RLE, this);
SetCounter("rle|longer", (RLE as RunLength01LongShortCodec).Counter_Longers);
SetCounter("rle|muchlonger", (RLE as RunLength01LongShortCodec).Counter_MuchLongers);
// Write size
DeltaTracker pos = new DeltaTracker();
output.WriteUInt32Optim((uint)image.Width);
output.WriteUInt32Optim((uint)image.Height);
SetCounter("bytes|size", pos.Next(output.Position));
// Write probs
ulong[] probs = CodecUtil.CountValues(fields, RLE.MaxSymbol);
CodecUtil.SaveFreqs(output, probs, TimwiCecCompressor.runLProbsProbs, "");
SetCounter("bytes|probs", pos.Next(output.Position));
// Write fields
ArithmeticWriter aw = new ArithmeticWriter(output, probs);
output.WriteUInt32Optim((uint)fields.Length);
foreach (var sym in fields)
{
aw.WriteSymbol(sym);
}
aw.Flush();
SetCounter("bytes|fields", pos.Next(output.Position));
}
public int PredictionEnTransformXor(Foreseer foreseer)
{
IntField orig = this.Clone();
foreseer.Initialize(orig);
int wrongPredictions = 0;
int p = 0;
for (int y = 0; y < Height; y++)
{
for (int x = 0; x < Width; x++, p++)
{
int predicted = foreseer.Foresee(orig, x, y);
Data[p] = orig.Data[p] ^ predicted;
if (Data[p] != 0)
{
wrongPredictions++;
}
foreseer.Learn(orig, x, y, orig.Data[p], predicted);
}
}
return(wrongPredictions);
}
public override IntField Decode(Stream input)
{
// Read size
int w = (int)input.ReadUInt32Optim();
int h = (int)input.ReadUInt32Optim();
// Read probabilities
ulong[] probs = CodecUtil.LoadFreqsCrappy(input, FieldcodeSymbols + 1);
// Read fields
ArithmeticSectionsCodec ac = new ArithmeticSectionsCodec(probs, 6);
ac.Decode(input);
var fields = new List <int[]>();
for (int i = 1; i <= 3; i++)
{
fields.Add(ac.ReadSection());
}
// Undo fieldcode
IntField transformed = CodecUtil.FieldcodeRunlengthsDe(fields, w, h, new RunLength01MaxSmartCodec(FieldcodeSymbols), this);
// Undo predictive transform
transformed.PredictionDeTransformXor(Seer);
transformed.ArgbFromField(0, 3);
return(transformed);
}
public static IntField FieldcodeRunlengthsDe(List <int[]> fields, int width, int height, SymbolCodec runlengthCodec, Compressor compr)
{
IntField image = new IntField(width, height);
for (int i = 1; i <= 3; i++)
{
CodecUtil.Shift(fields[i - 1], -1);
var f = runlengthCodec.Decode(fields[i - 1]);
for (int p = 0; p < width * height; p++)
{
if (f[p] == 1)
{
image.Data[p] = i;
}
}
// Visualise
IntField img = new IntField(width, height);
img.Data = f;
compr.AddImageGrayscale(img, 0, 1, "field" + i);
}
compr.AddImageGrayscale(image, 0, 3, "xformed");
return(image);
}
public override IntField Decode(Stream input)
{
// Read size
int w = (int)input.ReadUInt32Optim();
int h = (int)input.ReadUInt32Optim();
// Read probabilities
ulong[] probs = CodecUtil.LoadFreqs(input, TimwiCecCompressor.runLProbsProbs, RLE.MaxSymbol + 1);
// Read fields
int len = (int)input.ReadUInt32Optim();
ArithmeticCodingReader acr = new ArithmeticCodingReader(input, probs);
int[] fields = new int[len];
for (int p = 0; p < len; p++)
{
fields[p] = acr.ReadSymbol();
}
// Undo fieldcode
IntField transformed = CodecUtil.FieldcodeRunlengthsDe2(fields, w, h, RLE, this);
// Undo predictive transform
transformed.PredictionDeTransformXor(Seer);
transformed.ArgbFromField(0, 3);
return(transformed);
}
public IntField ReduceKeepingPixels(int blocksize)
{
IntField img = new IntField((Width + blocksize - 1) / blocksize, (Height + blocksize - 1) / blocksize);
for (int y = 0; y < img.Height; y++)
{
for (int x = 0; x < img.Width; x++)
{
int ix_end = Math.Min(x * blocksize + blocksize, Width);
int iy_end = Math.Min(y * blocksize + blocksize, Height);
for (int iy = y * blocksize; iy < iy_end; iy++)
{
for (int ix = x * blocksize; ix < ix_end; ix++)
{
if (this[ix, iy] != 0)
{
img[x, y] = 1;
goto breakout;
}
}
}
breakout :;
}
}
return(img);
}
public override void Learn(IntField image, int x, int y, int actual, int predicted)
{
foreach (var size in _sizes)
{
ulong hash = HashImageRegion(image, x, y, size.Width, size.Height);
_hashtable.AddCount(hash, actual);
}
}
public IntField Clone()
{
IntField result = (IntField)MemberwiseClone();
result.Data = new int[result.Data.Length];
Array.Copy(Data, result.Data, Data.Length);
return(result);
}
public IntField Extract(int fx, int fy, int width, int height)
{
IntField result = new IntField(width, height);
for (int y = fy; y < fy + height; y++)
{
Array.Copy(Data, fx + y * Width, result.Data, (y - fy) * width, width);
}
return(result);
}
public static List <Point> GetPixelCoords(IntField image, int color)
{
var indices = image.Data.Select((pix, ntx) => pix == color ? ntx : -1).Where(val => val >= 0);
var result = new List <Point>();
foreach (var index in indices)
{
result.Add(new Point(index % image.Width, index / image.Width));
}
return(result);
}
public static void AddImageTab(IntField argb, string caption)
{
if (TheInstance == null) // we're running in command line mode
{
return;
}
lock (Images)
{
Images.Enqueue(new Tuple <string, IntField>(caption, argb));
}
}
public IntField HalfResNearestNeighbour()
{
IntField result = new IntField((Width + 1) / 2, (Height + 1) / 2);
for (int x = 0; x < Width; x += 2)
{
for (int y = 0; y < Height; y += 2)
{
result[x >> 1, y >> 1] = this[x, y];
}
}
return(result);
}
/// <summary>
/// Decompresses a single file using a given compressor, saving all debug or
/// visualisation outputs to the specified directory.
/// </summary>
public static void DecompressFile(Compressor compr, string sourcePath, string destDir, string destFile)
{
FileStream input = File.Open(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read);
IntField f = compr.Decode(input);
input.Close();
Directory.CreateDirectory(destDir);
f.ArgbToBitmap().Save(Path.Combine(destDir, destFile), ImageFormat.Png);
SaveComprImages(compr, destDir);
SaveComprDumps(compr, destDir);
SaveComprCounters(compr, destDir);
}
public IntField HalfResHighestCount()
{
IntField result = new IntField((Width + 1) / 2, (Height + 1) / 2);
for (int x = 0; x < Width; x += 2)
{
for (int y = 0; y < Height; y += 2)
{
result[x >> 1, y >> 1] = MostFrequent(this[x, y], this[x + 1, y], this[x, y + 1], this[x + 1, y + 1]);
}
}
return(result);
}
private pattern findPattern(IntField image, int fx, int fy, int tw, int th)
{
int c = image[fx, fy];
int h = image[fx + 1, fy];
int v = image[fx, fy + 1];
bool certainlyNotCH = c != h;
bool certainlyNotCV = c != v;
bool certainlyNotSolid = c != h || c != v;
for (int y = fy + 1; y < fy + th; y++)
{
for (int x = fx + 1; x < fx + tw; x++)
{
c = image[x, y];
h = image[x - 1, y];
v = image[x, y - 1];
if (!certainlyNotSolid && (c != h || c != v))
{
certainlyNotSolid = true;
}
if (!certainlyNotCH && (c != h))
{
certainlyNotCH = true;
}
if (!certainlyNotCV && (c != v))
{
certainlyNotCV = true;
}
if (certainlyNotCH && certainlyNotCV && certainlyNotSolid)
{
return(pattern.Dirty);
}
}
}
if (!certainlyNotSolid)
{
return(pattern.Solid);
}
else if (!certainlyNotCV)
{
return(pattern.CleanVert);
}
else
{
return(pattern.CleanHorz);
}
}
public override void Encode(IntField image, Stream output)
{
/// Split background from foreground. Anything with low "rectangularity" goes into foreground
/// - measure the number of "turns" (changes from horz to vert edge) against area.
/// Background:
/// - Predictive xform with small radius
/// - Modified runlengths
/// - Context-aware arithmetic encoding
/// Foreground:
/// - Order colors by how common they are, call them c1,c2,c3
/// - Flatten all colors into 0 and 1
/// - Find all rects covering all c2-pixels without covering any c1-pixels.
/// - Find all rects covering all c3-pixels without covering any c1 or c2 pixels.
/// - Predictive xform on 0/1 image stopping at vertical letter boundaries. If larger image not in dict, use a smaller one and add all sizes to dict.
/// - Modified runlengths on the 0/1 xformed
/// - Context-aware arithmetic encoding
/// - Rects: optimize by changing 1 pixel rects to pixels; then feed optim ints through arithmetic
/// Modified runlengths:
/// - As before, encode only runs of 0's.
/// - Every run longer than thresh encoded as thresh + a number in a separate optim stream
/// - Optim stream is fed through arithmetic
///
/// Alternative color encoding: a stream of 1's, 2's, 3's, fed through runlength
image.ArgbTo4c();
AddImageGrayscale(image, 0, 3, "00-original");
IntField backgr, foregr;
backgroundSplit(image, out backgr, out foregr);
// Backgr
backgr.PredictionEnTransformXor(new FixedSizeForeseer(5, 3, 2, new HorzVertForeseer()));
AddImageGrayscale(backgr, 0, 3, "20-bkg-xformed");
saveColors(foregr);
// save as much as possible using rects, the rest just dumping as-is
// Foregr
IntField foremap = foregr.Clone();
foremap.Map(pix => pix == 0 ? 0 : 1);
AddImageGrayscale(foremap, 0, 1, "30-fore-map");
foremap.PredictionEnTransformXor(new FixedSizeForeseer(7, 7, 3, new HorzVertForeseer()));
AddImageGrayscale(foremap, 0, 1, "31-fore-map-xformed");
saveForeground(foremap.Data);
}
public override void Learn(IntField image, int x, int y, int actual, int predicted)
{
if (_curArea == null)
{
_fallback.Learn(image, x, y, actual, predicted);
return;
}
if (!_history.ContainsKey(_curArea))
{
_history.Add(_curArea, new ColorFrequencies());
}
_history[_curArea].Learn(actual);
}
public void Transpose()
{
var result = new IntField(Height, Width);
for (int x = 0; x < Width; x++)
{
for (int y = 0; y < Height; y++)
{
result[y, x] = this[x, y];
}
}
Width = result.Width;
Height = result.Height;
Data = result.Data;
}
/// <summary>
/// Compresses a single file using a given compressor, saving all debug or
/// visualisation outputs to the specified directory.
/// </summary>
public static void CompressFile(Compressor compr, string sourcePath, string destDir, string destFile)
{
IntField image = new IntField(0, 0);
image.ArgbLoadFromFile(sourcePath);
Directory.CreateDirectory(destDir);
compr.AddImageArgb(image, "orig");
using (var output = File.Open(Path.Combine(destDir, destFile), FileMode.Create, FileAccess.Write, FileShare.Read))
compr.Encode(image, output);
SaveComprImages(compr, destDir);
SaveComprDumps(compr, destDir);
SaveComprCounters(compr, destDir);
}
public override void Learn(IntField image, int x, int y, int actual, int predicted)
{
_fallback.Learn(image, x, y, actual, predicted);
foreach (var area in _curAreas)
{
ColorFrequencies cf;
if (!_history.TryGetValue(area, out cf))
{
_history.Add(area, cf = new ColorFrequencies());
}
cf.Learn(actual);
}
}
public override void Encode(IntField image, Stream output)
{
image.ArgbTo4c();
AddImageGrayscale(image, "res0");
List <IntField> scales = new List <IntField>();
scales.Add(image);
while (scales.Last().Width > 100 && scales.Last().Height > 100)
{
scales.Add(scales.Last().HalfResHighestCount());
}
for (int i = 1; i < scales.Count; i++)
{
IntField predicted = new IntField(scales[i - 1].Width, scales[i - 1].Height);
IntField shrunk = scales[i];
AddImageGrayscale(shrunk, "res" + i);
for (int y = 0; y < predicted.Height; y++)
{
for (int x = 0; x < predicted.Width; x++)
{
int xm2 = x & 1;
int ym2 = y & 1;
int xd2 = x >> 1;
int yd2 = y >> 1;
if (xm2 == 0 && ym2 == 0) // original
{
predicted[x, y] = shrunk[xd2, yd2];
}
else if (ym2 == 0) // between horizontal pixels
{
predicted[x, y] = shrunk[xd2, yd2];
}
else
{
predicted[x, y] = shrunk[xd2, yd2];
}
}
}
//AddImageGrayscale(predicted, "pred"+i);
for (int p = 0; p < predicted.Data.Length; p++)
{
predicted.Data[p] ^= scales[i - 1].Data[p];
}
//AddImageGrayscale(predicted, "diff"+i);
}
}
public static int[] FieldcodeRunlengthsEn2(IntField image, SymbolCodec runlengthCodec, Compressor compr)
{
compr.AddImageGrayscale(image, 0, 3, "xformed");
List <int> data = new List <int>();
for (int i = 1; i <= 3; i++)
{
IntField temp = image.Clone();
temp.Map(x => x == i ? 1 : 0);
data.AddRange(temp.Data);
compr.AddImageGrayscale(temp, 0, 1, "field" + i);
}
var runs = runlengthCodec.Encode(data.ToArray());
compr.SetCounter("runs", runs.Length);
return(runs);
}
public static List <int[]> FieldcodeRunlengthsEn(IntField image, SymbolCodec runlengthCodec, Compressor compr)
{
compr.AddImageGrayscale(image, 0, 3, "xformed");
var fields = new List <int[]>();
for (int i = 1; i <= 3; i++)
{
IntField temp = image.Clone();
temp.Map(x => x == i ? 1 : 0);
var field = runlengthCodec.Encode(temp.Data);
CodecUtil.Shift(field, 1);
compr.SetCounter("symbols|field-" + i, field.Length);
fields.Add(field);
compr.AddImageGrayscale(temp, 0, 1, "field" + i);
}
return(fields);
}