public static List<Rectangle> CleanupKerning(Bytemap sourceFrame, bool number) { byte[] letterBuf = sourceFrame.Bytes; Rectangle frame = sourceFrame.Frame; List<Rectangle> letters = new List<Rectangle>(); letters.Add(frame); for (int i = 0; i < letterBuf.Length; i++) { letterBuf[i] = (letterBuf[i] != 0) ? (byte)1 : (byte)0; } for (int i = 0; i < letters.Count; i++) { int max = 0; byte targetValue = (byte)(i + 2); // For a small interval, paint all pixels with value 1 using the // target color. This gives the flood fill something to start with. int fillUntil = (letters[i].X - frame.X) + Properties.Settings.Default.KerningEliminationAutoFill; fillUntil = Math.Min(fillUntil, frame.Width); for (int a = letters[i].X - frame.X; a < fillUntil; a++) { for (int b = 0; b < frame.Height; b++) { if (letterBuf[frame.Width * b + a] == 1) { letterBuf[frame.Width * b + a] = targetValue; max = Math.Max(max, a); } } } // Flood fill to replace 1 with the target color. FloodFill(1, targetValue, frame.Size, letterBuf, number); // Find rightmost column painted in target color.. for (int a = letters[i].X - frame.X; a < frame.Width; a++) { for (int b = 0; b < frame.Height; b++) { if (letterBuf[frame.Width * b + a] == targetValue) { max = Math.Max(max, a); } } } if (max != 0 && frame.Width - max > 2) { Rectangle r1 = GetSubRect(frame, letterBuf, 1); Rectangle r2 = GetSubRect(frame, letterBuf, targetValue); letters[i] = r2; letters.Insert(i + 1, r1); } } return letters; }
// Dump byte data from a (grayscale) bitmap to pixel array with // one byte per pixel. internal static unsafe Bytemap ExtractBytes(Bitmap source) { BitmapData data = source.LockBits(new Rectangle(0, 0, source.Width, source.Height), ImageLockMode.ReadWrite, source.PixelFormat); Bytemap bytemap = new Bytemap(new Rectangle(0, 0, source.Width, source.Height)); byte[] bytes = bytemap.Bytes; try { byte* d = (byte*)data.Scan0; for (int i = 0; i < source.Height; i++) { for (int j = 0; j < source.Width; j++) { bytes[i * source.Width + j] = d[4 * (i * data.Width + j)]; } } } finally { source.UnlockBits(data); } return bytemap; }
private void ApplyLetterMask(int n, Bytemap letter, Bytemap mask) { Rectangle letterFrame = letter.Frame; byte[] maskBytes = mask.Bytes; int DimensionX = letter.Frame.Width; n += 2; byte[] matchMask = new byte[letter.Bytes.Length]; // Relative position of letter / letterBytes in mask / letterMask. int diffX = letter.Frame.X-mask.Frame.X; int diffY = letter.Frame.Y - mask.Frame.Y; int untilY = Math.Min(mask.Frame.Height - diffY, letter.Frame.Height); int untilX = Math.Min(mask.Frame.Width - diffX, letter.Frame.Width); int letterW = letter.Frame.Width; int maskW = mask.Frame.Width; // Set match mask to pixels with correct value of n. // i, j index the matchMask, a,b index the letterMask for (int i = 0, a = diffY; i < untilY; i++, a++) { for (int j = 0, b = diffX; j < untilX; j++, b++) { matchMask[i * letterW + j] = (maskBytes[a * maskW + b] == n) ? (byte)1 : (byte)0; } } // Grow the blob of 1's in the match mask to include neighbouring pixels. for (int i = 0; i < untilY; i++) { for (int j = 0; j < untilX; j++) { if (matchMask[i * letterW + j] != 1) { continue; } for (int a = -2; a <= 2; a++) { for (int b = -2; b <= 2; b++) { if (a + i < 0 || a + i >= untilY) continue; if (b + j < 0 || b + j >= untilX) continue; int index = (i + a) * letterW + (j + b); if (index >= 0 && index < matchMask.Length) { if (matchMask[index] == 1) continue; matchMask[index] = 2; } } } } } for (int i = 0; i < untilY; i++) { for (int j = 0; j < untilX; j++) { if (matchMask[i * letterW + j] == 0) { letter.Bytes[i * DimensionX + j] = 0; } } } }
/// <summary> /// Copy a portion of an existing Bytemap into a new Bytemap. The new bytemap /// will always have a size of 'size' (typically a NN letter) regardless /// of the size of area. Pruning and zero-padding are used to ensure this. /// No data outside 'area' is copied. /// The rectangles that describe the position of the source and the selected area /// are all relative to the original page bitmap that the OCR is processing. /// </summary> internal static Bytemap CopyLetter(Bytemap source, Rectangle area, Size size) { byte[] sourceBytes = source.Bytes; int sourceW = source.Frame.Width; int dimX = size.Width; int dimY = size.Height; Bytemap bytemap = new Bytemap(new Rectangle(area.X, area.Y, dimX, dimY)); byte[] bytes = bytemap.Bytes; int min = 255; int diffX = area.X - source.Frame.X; int diffY = area.Y - source.Frame.Y; int untilY = Math.Min(source.Frame.Height - diffY, dimY); untilY = Math.Min(area.Height, dimY); int untilX = Math.Min(source.Frame.Width - diffX, dimX); untilX = Math.Min(area.Width, dimX); for (int i = 0; i < untilY; i++) { for (int j = 0; j < untilX; j++) { byte b = sourceBytes[(i + diffY) * sourceW + (j + diffX)]; bytes[i * dimX + j] = b; min = Math.Min(b, min); } } for (int i = 0; i < bytes.Length; i++) { bytes[i] = (byte)Math.Max(0, ((int)bytes[i] - (int)min)); } return bytemap; }
internal void ReadPage(Bytemap imageGray, Bytemap imageBinary, Bytemap imageBinarySplit, PageSections sections) { int descriptionLimit = -1; qualityData.Clear(); List<TransferItem> output = new List<TransferItem>(); // Get rid of those pesky powerplay tables. foreach (IPageSection section in sections.AllSections) { if (section is TextSection) { descriptionLimit = section.Bounds.Bottom; } } foreach (IPageSection section in sections.AllSections) { if (section is HeadlineSection) { TransferItem ti = ReadHeadline(section as HeadlineSection, imageGray, imageBinary); if (ti != null) { output.Add(ti); } } if (section is TextSection) { output.Add(ReadDescription(section as TextSection, imageGray, imageBinary)); } if (section is TableSection) { if (descriptionLimit > section.Bounds.Top) { continue; } output.AddRange(ReadTableSection(section as TableSection, sections, imageGray, imageBinary, imageBinarySplit)); } if (section is TextLineSection) { TransferItem ti; TextLineSection tsl = section as TextLineSection; ti = ReadTerraformingLine(tsl, sections, imageGray, imageBinary); if (ti != null) { output.Add(ti); } ti = ReadMiningReservesLine(tsl, sections, imageGray, imageBinary); if (ti != null) { output.Add(ti); } } } CustomItemProcessing(output); if (StitchPrevious) { output = MergeItems(currentItems, output); } AppendMetaInformation(output); currentItems = output.ToArray(); }
private string PredictAsLetterH(Bytemap imageGray, Bytemap imageBinary, Rectangle letter) { double quality; string text = ""; Bytemap letterMask = ImageLetters.CopyRectangle(imageBinary, letter); List<Rectangle> tmp = ImageLetters.CleanupKerning(letterMask, false); for (int i = 0; i < tmp.Count; i++) { Bytemap letterBytes = ImageLetters.CopyLetter(imageGray, tmp[i], nnHeadlines.InputSize); ApplyLetterMask(i, letterBytes, letterMask); text += nnHeadlines.Predict(letterBytes.Bytes, false, out quality); AddQualityData(quality, tmp[i]); } return text; }
private TransferItem ReadTableItem(Bytemap imageGray, Bytemap imageBinary, Bytemap imageBinarySplit, List<Line> left, List<Line> right) { List<Rectangle> allLeft = new List<Rectangle>(); foreach (Line line in left) { allLeft.AddRange(line); } string leftText = ""; for (int i = 0; i < allLeft.Count; i++) { if (i > 0 && ImageLetters.IsNewWord(allLeft, i, false)) { leftText += " "; } leftText += PredictAsLetter(imageGray, imageBinary, allLeft[i]); } TableItem item = GetTableItem(leftText); if (item == null) { return null; } if (RawMode) { item.Name = leftText; } TransferItem ti = new TransferItem(item.Name); for (int i = 0; i < right.Count; i++) { TransferItemValue tv = new TransferItemValue(); List<Rectangle> rightLine = new List<Rectangle>(right[i]); if (rightLine.Count == 0) continue; if (item.InitialSkip > 0) { rightLine.RemoveRange(0, Math.Min(item.InitialSkip, rightLine.Count)); } int numberLength = GetNumberLength(rightLine, item); string accumulateText = ""; string accumulateNumber = ""; for (int j = 0; j < rightLine.Count; j++) { if (accumulateText != "" && ImageLetters.IsNewWord(rightLine, j, false)) { accumulateText += " "; } // Test if in range for numerical or for text portion. if (j < numberLength) { if (j == 0 && rightLine[j].Width > 10) { // Extra splittish to get any "-" separated from the digit. accumulateNumber += PredictAsNumber(imageGray, imageBinarySplit, rightLine[j]); } else { accumulateNumber += PredictAsNumber(imageGray, imageBinary, rightLine[j]); } } else { // Reading part of text information: Read as letter. accumulateText += PredictAsLetter(imageGray, imageBinary, rightLine[j]); } } if (accumulateNumber != "") { tv.Value = GetNumericalValue(accumulateNumber, item.Percentage); } else { tv.Value = float.NaN; } if (accumulateText != "") { tv.Text = SimilarityMatch.GuessWords(accumulateText, wordList); } if (RawMode) { tv.Text = accumulateNumber + accumulateText; } tv.Unit = item.Unit; if (item.NoText) { tv.Text = ""; } // Special case of '<' in age table item. Don't know how to handle this systematically yet. if (item.Name == "AGE" && accumulateText.Split(new char[] { ' ' }).Length == 3) { tv.Value = 0; } ti.Values.Add(tv); } return ti; }
internal void ReadPageClassic(Bytemap imageGray, Bytemap imageBinary, Bytemap imageBinarySplit, PageSections sections) { qualityData.Clear(); List<TransferItem> output = new List<TransferItem>(); if (sections.DescriptiveText != null) { output.Add(ReadDescription(sections.DescriptiveText, imageGray, imageBinary)); } foreach (HeadlineSection hl in sections.Headlines) { TransferItem ti = ReadHeadline(hl, imageGray, imageBinary); if (ti != null) { output.Add(ti); } } foreach (TableSection table in sections.Tables) { output.AddRange(ReadTableSection(table, sections, imageGray, imageBinary, imageBinarySplit)); } foreach (TextLineSection tsl in sections.TextLines) { TransferItem ti; ti = ReadTerraformingLine(tsl, sections, imageGray, imageBinary); if (ti != null) { output.Add(ti); } ti = ReadMiningReservesLine(tsl, sections, imageGray, imageBinary); if (ti != null) { output.Add(ti); } } CustomItemProcessing(output); if (StitchPrevious) { output = MergeItems(currentItems, output); } AppendMetaInformation(output); currentItems = output.ToArray(); }
private TransferItem ReadTerraformingLine(TextLineSection tsl, PageSections sections, Bytemap imageGray, Bytemap imageBinary) { // Terraforming description is above the first table. if (sections.Tables.Count < 1) return null; if (tsl.Line.Bounds.Bottom >= sections.Tables[0].Bounds.Top) return null; string terraforming = ""; List<Rectangle> rs = new List<Rectangle>(tsl.Line); for (int i = 0; i < tsl.Line.Count; i++) { if (i > 0 && ImageLetters.IsNewWord(rs, i, true)) { terraforming += " "; } terraforming += PredictAsLetterD(imageGray, imageBinary, rs[i]); } return GuessTerraforming(terraforming); }
private TransferItem ReadDescription(TextSection desc, Bytemap imageGray, Bytemap imageBinary) { string description = ""; List<Rectangle> all = new List<Rectangle>(); for (int i = 0; i < desc.Count; i++) { all.AddRange(desc[i]); } for (int i = 0; i < all.Count; i++) { if (i > 0 && ImageLetters.IsNewWord(all, i, true)) { description += " "; } description += PredictAsLetterD(imageGray, imageBinary, all[i]); } TransferItem ti = new TransferItem(WellKnownItems.Description); TransferItemValue tv = new TransferItemValue(); if (!RawMode) { tv.Text = GuessDescription(description); } else { tv.Text = description; } tv.Value = float.NaN; ti.Values.Add(tv); return ti; }
private TransferItem ReadMiningReservesLine(TextLineSection tsl, PageSections sections, Bytemap imageGray, Bytemap imageBinary) { bool afterTable = false; bool afterDescription = false; // Mining reserves are stated between first table and a headline OR immediately after the description.. if (sections.Tables.Count < 1) return null; afterTable = tsl.Line.Bounds.Top > sections.Tables[0].Bounds.Bottom; afterDescription = tsl.Line.Bounds.Bottom < sections.Tables[0].Bounds.Top && tsl.Line.Bounds.Bottom > sections.Tables[0].Bounds.Top - 50; if (!afterTable && !afterDescription) return null; int index = sections.AllSections.IndexOf(tsl); if (index < 0 || index + 1 >= sections.AllSections.Count) return null; if (!afterDescription && !(sections.AllSections[index + 1] is HeadlineSection)) return null; string mining = ""; List<Rectangle> rs = new List<Rectangle>(tsl.Line); for (int i = 0; i < tsl.Line.Count; i++) { if (i > 0 && ImageLetters.IsNewWord(rs, i, true)) { mining += " "; } mining += PredictAsLetterD(imageGray, imageBinary, rs[i]); } return GuessMiningReserves(mining); }
private IEnumerable<TransferItem> ReadTableSection(TableSection table, PageSections sections, Bytemap imageGray, Bytemap imageBinary, Bytemap imageBinarySplit) { List<TransferItem> tis = new List<TransferItem>(); // TODO: For now, do not allow table items that are above the // description. This would be configurable in the future. if (sections.DescriptiveText != null && table.Bounds.Bottom < sections.DescriptiveText.Bounds.Top) { return tis; } if (!HasLeftText(table) || !HasRightText(table)) { return tis; } tis.Add(new TransferItem("DELIMITER")); for (int i = 0; i < table.Count; i++) { List<Line> left; List<Line> right; GetTableItem(table, i, out left, out right); tis.Add(ReadTableItem(imageGray, imageBinary, imageBinarySplit, left, right)); i += (left.Count - 1); } return tis; }
private TransferItem ReadHeadline(HeadlineSection hl, Bytemap imageGray, Bytemap imageBinary) { if (hl.Line.Count < 0) return null; if (hl.Line.Bounds.Height < 18) return null; if (hl.Line[0].Height < 20 && hl.Line[0].Width < 20) return null; string content = ""; List<Rectangle> rs = new List<Rectangle>(hl.Line); for (int i = 1 /* sic! */; i < hl.Line.Count; i++) { if (i > 1 /* sic! */ && ImageLetters.IsNewHeadlineWord(rs, i)) { content += " "; } content += PredictAsLetterH(imageGray, imageBinary, rs[i]); } TransferItem ti = new TransferItem(WellKnownItems.Headline); ti.Values.Add(new TransferItemValue()); ti.Values[0].Text = content; ti.Values[0].Value = double.NaN; return ti; }
internal static PageSections RefinePartition(PageSections pageSections, Bitmap binary) { Bytemap imageBinary = new Bytemap(binary); List<Line> descriptionLines = new List<Line>(); if (pageSections.DescriptiveText == null) { return pageSections; } foreach (Line line in pageSections.DescriptiveText) { List<Rectangle> accumulate = new List<Rectangle>(); foreach (Rectangle letter in line) { Bytemap letterMask = ImageLetters.CopyRectangle(imageBinary, letter); accumulate.AddRange(ImageLetters.CleanupKerning(letterMask, false)); } descriptionLines.Add(new Line(line.Bounds, accumulate)); } TextSection descriptiveText = new TextSection(descriptionLines); // Fix kerning for all text lines - hoping for terraforming and mining resources lines. List<TextLineSection> textLines = new List<TextLineSection>(); foreach (TextLineSection tls in pageSections.TextLines) { List<Rectangle> accumulate = new List<Rectangle>(); foreach (Rectangle letter in tls.Line) { Bytemap letterMask = ImageLetters.CopyRectangle(imageBinary, letter); accumulate.AddRange(ImageLetters.CleanupKerning(letterMask, false)); } textLines.Add(new TextLineSection(new Line(tls.Line.Bounds, accumulate))); } return new PageSections(pageSections.Tables, descriptiveText, textLines, pageSections.Excluded, pageSections.Headlines); }
private string PredictAsNumber(Bytemap imageGray, Bytemap imageBinary, Rectangle letter) { double quality; string text = ""; Bytemap letterMask = ImageLetters.CopyRectangle(imageBinary, letter); List<Rectangle> tmp = ImageLetters.CleanupKerning(letterMask, true); for (int i = 0; i < tmp.Count; i++) { // Reading part of number information: Read as digit. Bytemap letterBytes = ImageLetters.CopyLetter(imageGray, tmp[i], nnNumbers.InputSize); ApplyLetterMask(i, letterBytes, letterMask); char c; if (IsDelimiter(tmp[i])) { c = nnDelimiters.Predict(letterBytes.Bytes, false, out quality); } else { c = nnNumbers.Predict(letterBytes.Bytes, true, out quality); } if (c == '*') { // The current neural net is vulnurable against x/y offsets. Compensate. // TODO: Handle in a better way by adjusting the neural net. // TODO: Fix radius of planets e.g. 129, 223 Rectangle r2 = tmp[i]; r2.X += 1; letterBytes = ImageLetters.CopyLetter(imageGray, r2, nnNumbers.InputSize); ApplyLetterMask(i, letterBytes, letterMask); if (IsDelimiter(r2)) { c = nnDelimiters.Predict(ImageLetters.CopyLetter(imageGray, r2, nnNumbers.InputSize).Bytes, true, out quality); } else { c = nnNumbers.Predict(ImageLetters.CopyLetter(imageGray, r2, nnNumbers.InputSize).Bytes, true, out quality); } quality = Math.Max(0, quality - 0.25); } AddQualityData(quality, tmp[i]); FrmMain.DebugLog(c, tmp[i], letterBytes.Bytes); if (c == '*') { c = '.'; } text += c; } return text; }
/// <summary> /// Copy a portion of an existing Bytemap into a new Bytemap. The rectangles /// that describe the position of the source and the selected area are all /// relative to the original page bitmap that the OCR is processing. /// </summary> public static unsafe Bytemap CopyRectangle(Bytemap source, Rectangle area) { area.Intersect(source.Frame); if (area.IsEmpty) { return new Bytemap(new Rectangle(area.X, area.Y, Properties.Settings.Default.DimensionX, Properties.Settings.Default.DimensionY)); } byte[] sourceBytes = source.Bytes; int sourceW = source.Frame.Width; Bytemap dest = new Bytemap(area); byte[] destBytes = dest.Bytes; int diffX = area.X - source.Frame.X; int diffY = area.Y - source.Frame.Y; int untilY = Math.Min(source.Frame.Height - diffY, area.Height); int untilX = Math.Min(source.Frame.Width - diffX, area.Width); for (int i = 0; i < untilY; i++) { for (int j = 0; j < untilX; j++) { byte b = sourceBytes[(i + diffY) * sourceW + (j + diffX)]; destBytes[i * area.Width + j] = b; } } return dest; }
private TransferItem ReadTableItem(Bytemap imageGray, Bytemap imageBinary, Bytemap imageBinarySplit, List<Line> left, List<Line> right) { List<Rectangle> allLeft = new List<Rectangle>(); int lettersRight = 0; foreach (Line line in right) { lettersRight += line.Count; } if (lettersRight > 0 || left.Count <= 1) { foreach (Line line in left) { allLeft.AddRange(line); } } else { allLeft.AddRange(left[0]); for (int i = left.Count - 1; i >= 1; i--) { right.Insert(0, left[i]); } } string leftText = ""; for (int i = 0; i < allLeft.Count; i++) { if (i > 0 && ImageLetters.IsNewWord(allLeft, i, false)) { leftText += " "; } leftText += PredictAsLetter(imageGray, imageBinary, allLeft[i]); } TableItem item = GetTableItem(leftText); if (item == null) { return null; } if (RawMode) { item.Name = leftText; } TransferItem ti = new TransferItem(item.Name); for (int i = 0; i < right.Count; i++) { TransferItemValue tv = new TransferItemValue(); List<Rectangle> rightLine = new List<Rectangle>(right[i]); if (rightLine.Count == 0) continue; if (item.InitialSkip > 0) { rightLine.RemoveRange(0, Math.Min(item.InitialSkip, rightLine.Count)); } int numberLength = GetNumberLength(rightLine, item); string accumulateText = ""; string accumulateNumber = ""; for (int j = 0; j < rightLine.Count; j++) { if (accumulateText != "" && ImageLetters.IsNewWord(rightLine, j, false)) { accumulateText += " "; } // Test if in range for numerical or for text portion. if (j < numberLength) { if (j == 0 && rightLine[j].Width > 10) { // Extra splittish to get any "-" separated from the digit. accumulateNumber += PredictAsNumber(imageGray, imageBinarySplit, rightLine[j]); } else { accumulateNumber += PredictAsNumber(imageGray, imageBinary, rightLine[j]); } } else { // Reading part of text information: Read as letter. accumulateText += PredictAsLetter(imageGray, imageBinary, rightLine[j]); } } // TODO: Eliminate this hack. Should be a configurable flag in TableItem class. if (item.Name == "AGE" || item.Name == "RADIUS") { // Never has a decimal point, so the decimal point is most likely a ',' int indexDec = accumulateNumber.IndexOf('.'); if (indexDec >= 0 && accumulateNumber.IndexOf(',') < 0) { accumulateNumber = accumulateNumber.Substring(0, indexDec) + accumulateNumber.Substring(indexDec + 1); } } if (item.Name == "ORBIT_PERIAPSIS") { if (Math.Abs(GetNumericalValue(accumulateNumber, item.Percentage)) > 360) { accumulateNumber = accumulateNumber.Replace(',', '.'); } } if (accumulateNumber != "") { tv.Value = GetNumericalValue(accumulateNumber, item.Percentage); } else { tv.Value = float.NaN; } if (accumulateText != "") { tv.Text = SimilarityMatch.GuessWords(accumulateText, wordList); } if (RawMode) { tv.Text = accumulateNumber + accumulateText; } tv.Unit = item.Unit; if (item.NoText) { tv.Text = ""; } // Special case of '<' in age table item. Don't know how to handle this systematically yet. if (item.Name == "AGE" && accumulateText.Split(new char[] { ' ' }).Length == 3) { tv.Value = 0; } ti.Values.Add(tv); } return ti; }