public static List<Rectangle> CleanupKerning(Bytemap sourceFrame, bool number)
{
byte[] letterBuf = sourceFrame.Bytes;
Rectangle frame = sourceFrame.Frame;
List<Rectangle> letters = new List<Rectangle>();
letters.Add(frame);
for (int i = 0; i < letterBuf.Length; i++)
{
letterBuf[i] = (letterBuf[i] != 0) ? (byte)1 : (byte)0;
}
for (int i = 0; i < letters.Count; i++)
{
int max = 0;
byte targetValue = (byte)(i + 2);
// For a small interval, paint all pixels with value 1 using the
// target color. This gives the flood fill something to start with.
int fillUntil = (letters[i].X - frame.X) + Properties.Settings.Default.KerningEliminationAutoFill;
fillUntil = Math.Min(fillUntil, frame.Width);
for (int a = letters[i].X - frame.X; a < fillUntil; a++)
{
for (int b = 0; b < frame.Height; b++)
{
if (letterBuf[frame.Width * b + a] == 1)
{
letterBuf[frame.Width * b + a] = targetValue;
max = Math.Max(max, a);
}
}
}
// Flood fill to replace 1 with the target color.
FloodFill(1, targetValue, frame.Size, letterBuf, number);
// Find rightmost column painted in target color..
for (int a = letters[i].X - frame.X; a < frame.Width; a++)
{
for (int b = 0; b < frame.Height; b++)
{
if (letterBuf[frame.Width * b + a] == targetValue)
{
max = Math.Max(max, a);
}
}
}
if (max != 0 && frame.Width - max > 2)
{
Rectangle r1 = GetSubRect(frame, letterBuf, 1);
Rectangle r2 = GetSubRect(frame, letterBuf, targetValue);
letters[i] = r2;
letters.Insert(i + 1, r1);
}
}
return letters;
}
// Dump byte data from a (grayscale) bitmap to pixel array with
// one byte per pixel.
internal static unsafe Bytemap ExtractBytes(Bitmap source)
{
BitmapData data = source.LockBits(new Rectangle(0, 0, source.Width, source.Height), ImageLockMode.ReadWrite, source.PixelFormat);
Bytemap bytemap = new Bytemap(new Rectangle(0, 0, source.Width, source.Height));
byte[] bytes = bytemap.Bytes;
try
{
byte* d = (byte*)data.Scan0;
for (int i = 0; i < source.Height; i++)
{
for (int j = 0; j < source.Width; j++)
{
bytes[i * source.Width + j] = d[4 * (i * data.Width + j)];
}
}
}
finally
{
source.UnlockBits(data);
}
return bytemap;
}
private void ApplyLetterMask(int n, Bytemap letter, Bytemap mask)
{
Rectangle letterFrame = letter.Frame;
byte[] maskBytes = mask.Bytes;
int DimensionX = letter.Frame.Width;
n += 2;
byte[] matchMask = new byte[letter.Bytes.Length];
// Relative position of letter / letterBytes in mask / letterMask.
int diffX = letter.Frame.X-mask.Frame.X;
int diffY = letter.Frame.Y - mask.Frame.Y;
int untilY = Math.Min(mask.Frame.Height - diffY, letter.Frame.Height);
int untilX = Math.Min(mask.Frame.Width - diffX, letter.Frame.Width);
int letterW = letter.Frame.Width;
int maskW = mask.Frame.Width;
// Set match mask to pixels with correct value of n.
// i, j index the matchMask, a,b index the letterMask
for (int i = 0, a = diffY; i < untilY; i++, a++)
{
for (int j = 0, b = diffX; j < untilX; j++, b++)
{
matchMask[i * letterW + j] = (maskBytes[a * maskW + b] == n) ? (byte)1 : (byte)0;
}
}
// Grow the blob of 1's in the match mask to include neighbouring pixels.
for (int i = 0; i < untilY; i++)
{
for (int j = 0; j < untilX; j++)
{
if (matchMask[i * letterW + j] != 1)
{
continue;
}
for (int a = -2; a <= 2; a++)
{
for (int b = -2; b <= 2; b++)
{
if (a + i < 0 || a + i >= untilY) continue;
if (b + j < 0 || b + j >= untilX) continue;
int index = (i + a) * letterW + (j + b);
if (index >= 0 && index < matchMask.Length)
{
if (matchMask[index] == 1) continue;
matchMask[index] = 2;
}
}
}
}
}
for (int i = 0; i < untilY; i++)
{
for (int j = 0; j < untilX; j++)
{
if (matchMask[i * letterW + j] == 0)
{
letter.Bytes[i * DimensionX + j] = 0;
}
}
}
}
/// <summary>
/// Copy a portion of an existing Bytemap into a new Bytemap. The new bytemap
/// will always have a size of 'size' (typically a NN letter) regardless
/// of the size of area. Pruning and zero-padding are used to ensure this.
/// No data outside 'area' is copied.
/// The rectangles that describe the position of the source and the selected area
/// are all relative to the original page bitmap that the OCR is processing.
/// </summary>
internal static Bytemap CopyLetter(Bytemap source, Rectangle area, Size size)
{
byte[] sourceBytes = source.Bytes;
int sourceW = source.Frame.Width;
int dimX = size.Width;
int dimY = size.Height;
Bytemap bytemap = new Bytemap(new Rectangle(area.X, area.Y, dimX, dimY));
byte[] bytes = bytemap.Bytes;
int min = 255;
int diffX = area.X - source.Frame.X;
int diffY = area.Y - source.Frame.Y;
int untilY = Math.Min(source.Frame.Height - diffY, dimY);
untilY = Math.Min(area.Height, dimY);
int untilX = Math.Min(source.Frame.Width - diffX, dimX);
untilX = Math.Min(area.Width, dimX);
for (int i = 0; i < untilY; i++)
{
for (int j = 0; j < untilX; j++)
{
byte b = sourceBytes[(i + diffY) * sourceW + (j + diffX)];
bytes[i * dimX + j] = b;
min = Math.Min(b, min);
}
}
for (int i = 0; i < bytes.Length; i++)
{
bytes[i] = (byte)Math.Max(0, ((int)bytes[i] - (int)min));
}
return bytemap;
}
internal void ReadPage(Bytemap imageGray, Bytemap imageBinary, Bytemap imageBinarySplit, PageSections sections)
{
int descriptionLimit = -1;
qualityData.Clear();
List<TransferItem> output = new List<TransferItem>();
// Get rid of those pesky powerplay tables.
foreach (IPageSection section in sections.AllSections)
{
if (section is TextSection)
{
descriptionLimit = section.Bounds.Bottom;
}
}
foreach (IPageSection section in sections.AllSections)
{
if (section is HeadlineSection)
{
TransferItem ti = ReadHeadline(section as HeadlineSection, imageGray, imageBinary);
if (ti != null)
{
output.Add(ti);
}
}
if (section is TextSection)
{
output.Add(ReadDescription(section as TextSection, imageGray, imageBinary));
}
if (section is TableSection)
{
if (descriptionLimit > section.Bounds.Top)
{
continue;
}
output.AddRange(ReadTableSection(section as TableSection, sections, imageGray, imageBinary, imageBinarySplit));
}
if (section is TextLineSection)
{
TransferItem ti;
TextLineSection tsl = section as TextLineSection;
ti = ReadTerraformingLine(tsl, sections, imageGray, imageBinary);
if (ti != null)
{
output.Add(ti);
}
ti = ReadMiningReservesLine(tsl, sections, imageGray, imageBinary);
if (ti != null)
{
output.Add(ti);
}
}
}
CustomItemProcessing(output);
if (StitchPrevious)
{
output = MergeItems(currentItems, output);
}
AppendMetaInformation(output);
currentItems = output.ToArray();
}
private string PredictAsLetterH(Bytemap imageGray, Bytemap imageBinary, Rectangle letter)
{
double quality;
string text = "";
Bytemap letterMask = ImageLetters.CopyRectangle(imageBinary, letter);
List<Rectangle> tmp = ImageLetters.CleanupKerning(letterMask, false);
for (int i = 0; i < tmp.Count; i++)
{
Bytemap letterBytes = ImageLetters.CopyLetter(imageGray, tmp[i], nnHeadlines.InputSize);
ApplyLetterMask(i, letterBytes, letterMask);
text += nnHeadlines.Predict(letterBytes.Bytes, false, out quality);
AddQualityData(quality, tmp[i]);
}
return text;
}
private TransferItem ReadTableItem(Bytemap imageGray, Bytemap imageBinary, Bytemap imageBinarySplit, List<Line> left, List<Line> right)
{
List<Rectangle> allLeft = new List<Rectangle>();
foreach (Line line in left)
{
allLeft.AddRange(line);
}
string leftText = "";
for (int i = 0; i < allLeft.Count; i++)
{
if (i > 0 && ImageLetters.IsNewWord(allLeft, i, false))
{
leftText += " ";
}
leftText += PredictAsLetter(imageGray, imageBinary, allLeft[i]);
}
TableItem item = GetTableItem(leftText);
if (item == null)
{
return null;
}
if (RawMode)
{
item.Name = leftText;
}
TransferItem ti = new TransferItem(item.Name);
for (int i = 0; i < right.Count; i++)
{
TransferItemValue tv = new TransferItemValue();
List<Rectangle> rightLine = new List<Rectangle>(right[i]);
if (rightLine.Count == 0) continue;
if (item.InitialSkip > 0)
{
rightLine.RemoveRange(0, Math.Min(item.InitialSkip, rightLine.Count));
}
int numberLength = GetNumberLength(rightLine, item);
string accumulateText = "";
string accumulateNumber = "";
for (int j = 0; j < rightLine.Count; j++)
{
if (accumulateText != "" && ImageLetters.IsNewWord(rightLine, j, false))
{
accumulateText += " ";
}
// Test if in range for numerical or for text portion.
if (j < numberLength)
{
if (j == 0 && rightLine[j].Width > 10)
{
// Extra splittish to get any "-" separated from the digit.
accumulateNumber += PredictAsNumber(imageGray, imageBinarySplit, rightLine[j]);
}
else
{
accumulateNumber += PredictAsNumber(imageGray, imageBinary, rightLine[j]);
}
}
else
{
// Reading part of text information: Read as letter.
accumulateText += PredictAsLetter(imageGray, imageBinary, rightLine[j]);
}
}
if (accumulateNumber != "")
{
tv.Value = GetNumericalValue(accumulateNumber, item.Percentage);
}
else
{
tv.Value = float.NaN;
}
if (accumulateText != "")
{
tv.Text = SimilarityMatch.GuessWords(accumulateText, wordList);
}
if (RawMode)
{
tv.Text = accumulateNumber + accumulateText;
}
tv.Unit = item.Unit;
if (item.NoText)
{
tv.Text = "";
}
// Special case of '<' in age table item. Don't know how to handle this systematically yet.
if (item.Name == "AGE" && accumulateText.Split(new char[] { ' ' }).Length == 3)
{
tv.Value = 0;
}
ti.Values.Add(tv);
}
return ti;
}
internal void ReadPageClassic(Bytemap imageGray, Bytemap imageBinary, Bytemap imageBinarySplit, PageSections sections)
{
qualityData.Clear();
List<TransferItem> output = new List<TransferItem>();
if (sections.DescriptiveText != null)
{
output.Add(ReadDescription(sections.DescriptiveText, imageGray, imageBinary));
}
foreach (HeadlineSection hl in sections.Headlines)
{
TransferItem ti = ReadHeadline(hl, imageGray, imageBinary);
if (ti != null)
{
output.Add(ti);
}
}
foreach (TableSection table in sections.Tables)
{
output.AddRange(ReadTableSection(table, sections, imageGray, imageBinary, imageBinarySplit));
}
foreach (TextLineSection tsl in sections.TextLines)
{
TransferItem ti;
ti = ReadTerraformingLine(tsl, sections, imageGray, imageBinary);
if (ti != null)
{
output.Add(ti);
}
ti = ReadMiningReservesLine(tsl, sections, imageGray, imageBinary);
if (ti != null)
{
output.Add(ti);
}
}
CustomItemProcessing(output);
if (StitchPrevious)
{
output = MergeItems(currentItems, output);
}
AppendMetaInformation(output);
currentItems = output.ToArray();
}
private TransferItem ReadTerraformingLine(TextLineSection tsl, PageSections sections, Bytemap imageGray, Bytemap imageBinary)
{
// Terraforming description is above the first table.
if (sections.Tables.Count < 1) return null;
if (tsl.Line.Bounds.Bottom >= sections.Tables[0].Bounds.Top) return null;
string terraforming = "";
List<Rectangle> rs = new List<Rectangle>(tsl.Line);
for (int i = 0; i < tsl.Line.Count; i++)
{
if (i > 0 && ImageLetters.IsNewWord(rs, i, true))
{
terraforming += " ";
}
terraforming += PredictAsLetterD(imageGray, imageBinary, rs[i]);
}
return GuessTerraforming(terraforming);
}
private TransferItem ReadDescription(TextSection desc, Bytemap imageGray, Bytemap imageBinary)
{
string description = "";
List<Rectangle> all = new List<Rectangle>();
for (int i = 0; i < desc.Count; i++)
{
all.AddRange(desc[i]);
}
for (int i = 0; i < all.Count; i++)
{
if (i > 0 && ImageLetters.IsNewWord(all, i, true))
{
description += " ";
}
description += PredictAsLetterD(imageGray, imageBinary, all[i]);
}
TransferItem ti = new TransferItem(WellKnownItems.Description);
TransferItemValue tv = new TransferItemValue();
if (!RawMode)
{
tv.Text = GuessDescription(description);
}
else
{
tv.Text = description;
}
tv.Value = float.NaN;
ti.Values.Add(tv);
return ti;
}
private TransferItem ReadMiningReservesLine(TextLineSection tsl, PageSections sections, Bytemap imageGray, Bytemap imageBinary)
{
bool afterTable = false;
bool afterDescription = false;
// Mining reserves are stated between first table and a headline OR immediately after the description..
if (sections.Tables.Count < 1) return null;
afterTable = tsl.Line.Bounds.Top > sections.Tables[0].Bounds.Bottom;
afterDescription = tsl.Line.Bounds.Bottom < sections.Tables[0].Bounds.Top
&& tsl.Line.Bounds.Bottom > sections.Tables[0].Bounds.Top - 50;
if (!afterTable && !afterDescription) return null;
int index = sections.AllSections.IndexOf(tsl);
if (index < 0 || index + 1 >= sections.AllSections.Count) return null;
if (!afterDescription && !(sections.AllSections[index + 1] is HeadlineSection)) return null;
string mining = "";
List<Rectangle> rs = new List<Rectangle>(tsl.Line);
for (int i = 0; i < tsl.Line.Count; i++)
{
if (i > 0 && ImageLetters.IsNewWord(rs, i, true))
{
mining += " ";
}
mining += PredictAsLetterD(imageGray, imageBinary, rs[i]);
}
return GuessMiningReserves(mining);
}
private IEnumerable<TransferItem> ReadTableSection(TableSection table, PageSections sections, Bytemap imageGray, Bytemap imageBinary, Bytemap imageBinarySplit)
{
List<TransferItem> tis = new List<TransferItem>();
// TODO: For now, do not allow table items that are above the
// description. This would be configurable in the future.
if (sections.DescriptiveText != null && table.Bounds.Bottom < sections.DescriptiveText.Bounds.Top)
{
return tis;
}
if (!HasLeftText(table) || !HasRightText(table))
{
return tis;
}
tis.Add(new TransferItem("DELIMITER"));
for (int i = 0; i < table.Count; i++)
{
List<Line> left;
List<Line> right;
GetTableItem(table, i, out left, out right);
tis.Add(ReadTableItem(imageGray, imageBinary, imageBinarySplit, left, right));
i += (left.Count - 1);
}
return tis;
}
private TransferItem ReadHeadline(HeadlineSection hl, Bytemap imageGray, Bytemap imageBinary)
{
if (hl.Line.Count < 0) return null;
if (hl.Line.Bounds.Height < 18) return null;
if (hl.Line[0].Height < 20 && hl.Line[0].Width < 20) return null;
string content = "";
List<Rectangle> rs = new List<Rectangle>(hl.Line);
for (int i = 1 /* sic! */; i < hl.Line.Count; i++)
{
if (i > 1 /* sic! */ && ImageLetters.IsNewHeadlineWord(rs, i))
{
content += " ";
}
content += PredictAsLetterH(imageGray, imageBinary, rs[i]);
}
TransferItem ti = new TransferItem(WellKnownItems.Headline);
ti.Values.Add(new TransferItemValue());
ti.Values[0].Text = content;
ti.Values[0].Value = double.NaN;
return ti;
}
internal static PageSections RefinePartition(PageSections pageSections, Bitmap binary)
{
Bytemap imageBinary = new Bytemap(binary);
List<Line> descriptionLines = new List<Line>();
if (pageSections.DescriptiveText == null)
{
return pageSections;
}
foreach (Line line in pageSections.DescriptiveText)
{
List<Rectangle> accumulate = new List<Rectangle>();
foreach (Rectangle letter in line)
{
Bytemap letterMask = ImageLetters.CopyRectangle(imageBinary, letter);
accumulate.AddRange(ImageLetters.CleanupKerning(letterMask, false));
}
descriptionLines.Add(new Line(line.Bounds, accumulate));
}
TextSection descriptiveText = new TextSection(descriptionLines);
// Fix kerning for all text lines - hoping for terraforming and mining resources lines.
List<TextLineSection> textLines = new List<TextLineSection>();
foreach (TextLineSection tls in pageSections.TextLines)
{
List<Rectangle> accumulate = new List<Rectangle>();
foreach (Rectangle letter in tls.Line)
{
Bytemap letterMask = ImageLetters.CopyRectangle(imageBinary, letter);
accumulate.AddRange(ImageLetters.CleanupKerning(letterMask, false));
}
textLines.Add(new TextLineSection(new Line(tls.Line.Bounds, accumulate)));
}
return new PageSections(pageSections.Tables, descriptiveText, textLines, pageSections.Excluded, pageSections.Headlines);
}
private string PredictAsNumber(Bytemap imageGray, Bytemap imageBinary, Rectangle letter)
{
double quality;
string text = "";
Bytemap letterMask = ImageLetters.CopyRectangle(imageBinary, letter);
List<Rectangle> tmp = ImageLetters.CleanupKerning(letterMask, true);
for (int i = 0; i < tmp.Count; i++)
{
// Reading part of number information: Read as digit.
Bytemap letterBytes = ImageLetters.CopyLetter(imageGray, tmp[i], nnNumbers.InputSize);
ApplyLetterMask(i, letterBytes, letterMask);
char c;
if (IsDelimiter(tmp[i]))
{
c = nnDelimiters.Predict(letterBytes.Bytes, false, out quality);
}
else
{
c = nnNumbers.Predict(letterBytes.Bytes, true, out quality);
}
if (c == '*')
{
// The current neural net is vulnurable against x/y offsets. Compensate.
// TODO: Handle in a better way by adjusting the neural net.
// TODO: Fix radius of planets e.g. 129, 223
Rectangle r2 = tmp[i];
r2.X += 1;
letterBytes = ImageLetters.CopyLetter(imageGray, r2, nnNumbers.InputSize);
ApplyLetterMask(i, letterBytes, letterMask);
if (IsDelimiter(r2))
{
c = nnDelimiters.Predict(ImageLetters.CopyLetter(imageGray, r2, nnNumbers.InputSize).Bytes, true, out quality);
}
else
{
c = nnNumbers.Predict(ImageLetters.CopyLetter(imageGray, r2, nnNumbers.InputSize).Bytes, true, out quality);
}
quality = Math.Max(0, quality - 0.25);
}
AddQualityData(quality, tmp[i]);
FrmMain.DebugLog(c, tmp[i], letterBytes.Bytes);
if (c == '*')
{
c = '.';
}
text += c;
}
return text;
}
/// <summary>
/// Copy a portion of an existing Bytemap into a new Bytemap. The rectangles
/// that describe the position of the source and the selected area are all
/// relative to the original page bitmap that the OCR is processing.
/// </summary>
public static unsafe Bytemap CopyRectangle(Bytemap source, Rectangle area)
{
area.Intersect(source.Frame);
if (area.IsEmpty)
{
return new Bytemap(new Rectangle(area.X, area.Y, Properties.Settings.Default.DimensionX, Properties.Settings.Default.DimensionY));
}
byte[] sourceBytes = source.Bytes;
int sourceW = source.Frame.Width;
Bytemap dest = new Bytemap(area);
byte[] destBytes = dest.Bytes;
int diffX = area.X - source.Frame.X;
int diffY = area.Y - source.Frame.Y;
int untilY = Math.Min(source.Frame.Height - diffY, area.Height);
int untilX = Math.Min(source.Frame.Width - diffX, area.Width);
for (int i = 0; i < untilY; i++)
{
for (int j = 0; j < untilX; j++)
{
byte b = sourceBytes[(i + diffY) * sourceW + (j + diffX)];
destBytes[i * area.Width + j] = b;
}
}
return dest;
}
private TransferItem ReadTableItem(Bytemap imageGray, Bytemap imageBinary, Bytemap imageBinarySplit, List<Line> left, List<Line> right)
{
List<Rectangle> allLeft = new List<Rectangle>();
int lettersRight = 0;
foreach (Line line in right)
{
lettersRight += line.Count;
}
if (lettersRight > 0 || left.Count <= 1)
{
foreach (Line line in left)
{
allLeft.AddRange(line);
}
}
else
{
allLeft.AddRange(left[0]);
for (int i = left.Count - 1; i >= 1; i--)
{
right.Insert(0, left[i]);
}
}
string leftText = "";
for (int i = 0; i < allLeft.Count; i++)
{
if (i > 0 && ImageLetters.IsNewWord(allLeft, i, false))
{
leftText += " ";
}
leftText += PredictAsLetter(imageGray, imageBinary, allLeft[i]);
}
TableItem item = GetTableItem(leftText);
if (item == null)
{
return null;
}
if (RawMode)
{
item.Name = leftText;
}
TransferItem ti = new TransferItem(item.Name);
for (int i = 0; i < right.Count; i++)
{
TransferItemValue tv = new TransferItemValue();
List<Rectangle> rightLine = new List<Rectangle>(right[i]);
if (rightLine.Count == 0) continue;
if (item.InitialSkip > 0)
{
rightLine.RemoveRange(0, Math.Min(item.InitialSkip, rightLine.Count));
}
int numberLength = GetNumberLength(rightLine, item);
string accumulateText = "";
string accumulateNumber = "";
for (int j = 0; j < rightLine.Count; j++)
{
if (accumulateText != "" && ImageLetters.IsNewWord(rightLine, j, false))
{
accumulateText += " ";
}
// Test if in range for numerical or for text portion.
if (j < numberLength)
{
if (j == 0 && rightLine[j].Width > 10)
{
// Extra splittish to get any "-" separated from the digit.
accumulateNumber += PredictAsNumber(imageGray, imageBinarySplit, rightLine[j]);
}
else
{
accumulateNumber += PredictAsNumber(imageGray, imageBinary, rightLine[j]);
}
}
else
{
// Reading part of text information: Read as letter.
accumulateText += PredictAsLetter(imageGray, imageBinary, rightLine[j]);
}
}
// TODO: Eliminate this hack. Should be a configurable flag in TableItem class.
if (item.Name == "AGE" || item.Name == "RADIUS")
{
// Never has a decimal point, so the decimal point is most likely a ','
int indexDec = accumulateNumber.IndexOf('.');
if (indexDec >= 0 && accumulateNumber.IndexOf(',') < 0)
{
accumulateNumber = accumulateNumber.Substring(0, indexDec) + accumulateNumber.Substring(indexDec + 1);
}
}
if (item.Name == "ORBIT_PERIAPSIS")
{
if (Math.Abs(GetNumericalValue(accumulateNumber, item.Percentage)) > 360)
{
accumulateNumber = accumulateNumber.Replace(',', '.');
}
}
if (accumulateNumber != "")
{
tv.Value = GetNumericalValue(accumulateNumber, item.Percentage);
}
else
{
tv.Value = float.NaN;
}
if (accumulateText != "")
{
tv.Text = SimilarityMatch.GuessWords(accumulateText, wordList);
}
if (RawMode)
{
tv.Text = accumulateNumber + accumulateText;
}
tv.Unit = item.Unit;
if (item.NoText)
{
tv.Text = "";
}
// Special case of '<' in age table item. Don't know how to handle this systematically yet.
if (item.Name == "AGE" && accumulateText.Split(new char[] { ' ' }).Length == 3)
{
tv.Value = 0;
}
ti.Values.Add(tv);
}
return ti;
}
