private static void MergeGoogleOcr(FilledForm newForm, List <TextBox> googleResults) { foreach (var field in newForm.ImageFields.Where(x => x.FieldResult.FieldType == "Text" && !x.Field.Name.Contains("block"))) { if (field.FieldResult?.Text == null) { continue; } int bestScore = 10000; // lower is better TextBox bestGoogleBox = null; var perfectScores = new List <TextBox>(); foreach (var gr in googleResults) { var score = field.CalcRectangleMatchScore(gr.Rect); if (field.FieldResult.Text.Length > 2) { logger.Info( $"{score} [{gr.Description}] {gr.Bounds} {field.Rectangle} [{field.FieldResult.Text}]"); } if (score <= bestScore) { if (bestScore == 0) { logger.Info($"second perfect score"); } bestScore = score; bestGoogleBox = gr; if (score == 0) { perfectScores.Add(gr); } } } if (bestScore < 30) { field.FieldResult.GoogleConfidence = bestGoogleBox.Confidence * 100.0; field.FieldResult.GoogleText = bestGoogleBox.Description; if (perfectScores.Count > 1) { field.FieldResult.GoogleConfidence = perfectScores.Min(x => x.Confidence * 100.0); field.FieldResult.GoogleText = perfectScores.Select(x => x.Description).Aggregate("", (a, b) => a + " " + b); } } } }
public void CreateFormForRecognition(FilledForm form, FormsRecognitionMethod method) { form.Attributes = CreateForm(method); var image = form.GetImage().CloneAll(); //int saveCurrentPageIndex = image.Page; for (int i = 0; i < image.PageCount; i++) { image.Page = i + 1;//page index is a 1-based starts from 1 not zero PageRecognitionOptions pageOptions = new PageRecognitionOptions(); pageOptions.UpdateImage = true; pageOptions.PageType = FormsPageType.Normal; AddPageToForm(image, form.Attributes, pageOptions); } }
public void AlignForm(FilledForm form, bool calculateAlignment) { if (calculateAlignment) { CreateFormForRecognition(form, FormsRecognitionMethod.Complex); form.Alignment = RecognitionEngine.GetFormAlignment(form.Master.Attributes, form.Attributes, null); } else { form.Alignment = new List <PageAlignment>(); for (int i = 0; i < form.Result.PageResults.Count; i++) { form.Alignment.Add(form.Result.PageResults[i].Alignment); } } }
public void SaveFields(FilledForm form, List <OcrField> ocrFields, bool doOmr = false) { var sb = new StringBuilder(); sb.Append($"OMR={doOmr}\n"); sb.Append($"FormName: {form.Name}\n"); sb.Append($"FileName: {form.FileName}\n"); sb.Append(form.Master.Properties.Name + "\n"); var fi = new FileInfo(form.FileName); Debug.Assert(fi.DirectoryName != null, "fi.DirectoryName != null"); var outDir = Path.Combine(fi.DirectoryName, "data"); EnsurePathExists(outDir); var fiMaster = Path.GetFileNameWithoutExtension(form.Master.Properties.Name); var baseMergedName = Path.GetFileNameWithoutExtension(form.FileName) + "~" + fiMaster; var jsonMergedName = Path.Combine(outDir, baseMergedName + "_merged.json"); var baseName = Path.GetFileNameWithoutExtension(form.FileName) + (doOmr ? "_omr" : "_ocr") + "~" + fiMaster; var image = doOmr ? form.OmrImage : form.Image; Debug.Assert(outDir != null, "outDir != null"); var imageName = Path.Combine(outDir, baseName + ".png"); var jsonName = Path.Combine(outDir, baseName + ".json"); var jsonName2 = Path.Combine(outDir, baseName + "_short.json"); var fieldInfo = Path.Combine(outDir, baseName + ".field_info"); sb.Append($"imageName: {imageName}\n"); sb.Append($"jsonName: {jsonName}\n"); sb.Append($"jsonName2: {jsonName2}\n"); sb.Append($"fieldInfo: {fieldInfo}\n"); RasterCodecs.Save(image, imageName, RasterImageFormat.Png, 1); if (form.ProcessingPages?[0] != null) { var fields = new FormField[form.ProcessingPages[0].Count]; int i = 0; foreach (var field in form.ProcessingPages[0]) { fields[i++] = field; } sb.Append($"Json: {jsonName}"); var json = JsonConvert.SerializeObject(fields, Formatting.Indented); File.WriteAllText(jsonName, json); sb.Append(json); OcrRecord curRecord = new OcrRecord(); curRecord.AppID = AppID; curRecord.MasterForms.Add(form.Master.Properties.Name); sb = new StringBuilder(); var jArray = Newtonsoft.Json.Linq.JArray.Parse(json); //Debug.WriteLine(jArray.Count); foreach (var jToken in jArray) { var name = jToken.Value <string>("Name"); var bounds = jToken.Value <string>("Bounds"); var res = jToken["ResultDefault"]; if (res != null) { if (!doOmr) { var f = new OcrField(name, "text", bounds, res); Debug.Assert(curRecord != null, nameof(curRecord) + " != null"); ocrFields.Add(f); curRecord.AddField(f); } } else { if (doOmr) { res = jToken["Result"]; var f = new OcrField(name, "omr", bounds, res); Debug.Assert(curRecord != null, nameof(curRecord) + " != null"); ocrFields.Add(f); curRecord.AddField(f); } } } File.WriteAllText(jsonName2, JsonConvert.SerializeObject(curRecord, Formatting.Indented)); var fieldsSort = from o in ocrFields orderby o.VariableName select o; File.WriteAllText(jsonMergedName, JsonConvert.SerializeObject(fieldsSort.ToArray(), Formatting.Indented)); } else { sb.Append("Json: No Fields found\n"); } File.WriteAllText(fieldInfo, sb.ToString()); logger.Info(sb.ToString()); }
private FormRecognitionAttributes PrepareNewForm(ImageInfo ofi, Stopwatch stopWatch, FilledForm newForm) { try { FormRecognitionAttributes filledFormAttributes = RecognitionEngine.CreateForm(null); ofi.Image.ChangeViewPerspective(RasterViewPerspective.TopLeft); logger.Info($"{stopWatch.ElapsedMilliseconds} change view"); for (int i = 0; i < ofi.Image.PageCount; i++) { ofi.Image.Page = i + 1; //Add each page of the filled form to the recognition engine RecognitionEngine.AddFormPage(filledFormAttributes, ofi.Image, null); } logger.Info($"{stopWatch.ElapsedMilliseconds} closing form"); RecognitionEngine.CloseForm(filledFormAttributes); logger.Info($"{stopWatch.ElapsedMilliseconds} closed form"); CreateFormForRecognition(newForm, FormsRecognitionMethod.Complex); logger.Info($"{stopWatch.ElapsedMilliseconds} create form"); return(filledFormAttributes); } catch (ThreadAbortException) { logger.Error($"Thread aborted {stopWatch.ElapsedMilliseconds}"); } return(null); }
public List <FilledForm> ProcessOcr(ResultsForPrettyJson formResults, List <ImageInfo> fileInfos) { try { var outDir = formResults.OriginalDirectoryName; var retForms = new List <FilledForm>(); var usedMasters = new HashSet <MasterForm>(); Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); formResults.PagesInPdf = fileInfos.Count; foreach (var ofi in fileInfos) { FilledForm newForm = new FilledForm(); retForms.Add(newForm); newForm.ImageInfoMaster.InitialImage = ofi; newForm.Name = Path.GetFileNameWithoutExtension(ofi.ImageFileInfo.Name); if (ofi.Image == null) { ofi.Image = LoadImageFile(ofi.ImageFileInfo.FullName, 1, -1); } //CleanupImage(ofi.Image); var par = new FormThreadCallParams() { ImageInfo = ofi, StopWatch = stopWatch, Form = newForm }; if (PageTimeoutInSeconds < 50) { Thread t = new Thread(this.PrepareNewFormThreader); t.Start(par); if (!t.Join(TimeSpan.FromSeconds(PageTimeoutInSeconds))) { t.Abort(); formResults.TimedOutPages.Add(newForm.Name); formResults.BestFormConfidence.Add(-1); if (formResults.TimedOutPages.Count > 2 && formResults.PagesMappedToForm == 0) { formResults.Status = $"Form abandoned for timeout after {formResults.BestFormConfidence.Count} pages"; logger.Error(formResults.Status); return(retForms); } continue; } } else { PrepareNewFormThreader(par); } Debug.Assert(par.Attributes != null); var filledFormAttributes = par.Attributes; //List<FormRecognitionResult> results = new List<FormRecognitionResult>(); MasterForm currentMasterBlockForm = null; int bestConfidence = -1; int currentConfidence = 85; foreach (var master in BlockMasterForms) { if (usedMasters.Contains(master)) { continue; } var result = RecognitionEngine.CompareForm(master.Attributes, filledFormAttributes, null, null); //logger.Debug($"Check {master} for {newForm} {stopWatch.ElapsedMilliseconds} {result.Confidence}"); if (result.Confidence > currentConfidence) { currentMasterBlockForm = master; bestConfidence = currentConfidence = result.Confidence; } else if (result.Confidence > bestConfidence) { bestConfidence = result.Confidence; } } formResults.BestFormConfidence.Add(bestConfidence); if (currentMasterBlockForm != null) { formResults.MasterFormPages.Add(currentMasterBlockForm.Properties.Name); formResults.PagesMappedToForm++; logger.Info($"FilledForm matched {newForm.Name} {newForm.Status} {stopWatch.ElapsedMilliseconds} "); newForm.ImageInfoMaster.InitialImage = ofi; var centeredImage = ofi.Image.CloneAll(); CleanupImage(centeredImage); newForm.ImageInfoMaster.CenteredImage = new ImageInfo() { Image = centeredImage }; var omrImage = centeredImage.CloneAll(); PrepareOmrImage(omrImage); newForm.ImageInfoMaster.OmrImage = new ImageInfo() { Image = omrImage }; newForm.Status = "Matched"; newForm.Master = currentMasterBlockForm; var alignment = RecognitionEngine.GetFormAlignment(newForm.Master.Attributes, newForm.Attributes, null); var fields = currentMasterBlockForm.ProcessingPages[0]; var scaler = currentMasterBlockForm.Resolution; var fieldsOnlyImage = RasterImage.Create(centeredImage.Width, centeredImage.Height, centeredImage.BitsPerPixel, 300, RasterColor.White); //fieldsOnlyImage = new RasterImage(RasterMemoryFlags.Conventional, centeredImage.Width, centeredImage.Height, centeredInage.BitsPerPixel, RasterByteOrder.Rgb, RasterViewPerspective.TopLeft, null, null, 0); var subDirField = Path.Combine(outDir, "fields"); var fileNameFieldOnly = Path.Combine(subDirField, newForm.Name + "_fields.jpg"); var googleResultsFile = Path.Combine(subDirField, newForm.Name + "_google.json"); var combined = false; foreach (var field in fields) { var isBlock = field.Name.Contains("block"); var rect200 = alignment[0].AlignRectangle(field.Bounds); scaler = 300; int fudge = isBlock ? 30 : 1; var rect300 = new LeadRect(rect200.Left * 300 / scaler - fudge, rect200.Top * 300 / scaler - fudge, rect200.Width * 300 / scaler + fudge, rect200.Height * 300 / scaler + fudge); try { var imageInfoToUse = newForm.ImageInfoMaster.CenteredImage; var zoneType = OcrZoneType.Text; if (field.GetType() == typeof(OmrFormField)) { imageInfoToUse = newForm.ImageInfoMaster.OmrImage; zoneType = OcrZoneType.Omr; } else if (field.GetType() == typeof(ImageFormField)) { zoneType = OcrZoneType.Graphic; } var image = imageInfoToUse.Image.CloneAll(); var subDir = Path.Combine(outDir, isBlock ? "blocks" : "fields"); var fileName = Path.Combine(subDir, newForm.Name + "_" + field.Name + ".jpg"); var imageField = new ImageField { Field = field, FieldResult = { FieldName = field.Name, IsBlock = isBlock, ImageFile = fileName, Bounds = rect300.ToString(), FieldType = zoneType.ToString(), Error = "None" } }; imageField.Rectangle = new Rectangle(rect300.X, rect300.Y, rect300.Width, rect300.Height); try { EnsurePathExists(subDir); CropCommand command = new CropCommand { Rectangle = rect300 }; command.Run(image); RasterCodecs.Save(image, fileName, RasterImageFormat.Jpeg, bitsPerPixel: 8); if (!isBlock && zoneType == OcrZoneType.Text && !combined) { try { ; var combiner = new CombineCommand(); //combiner.DestinationImage = fieldsOnlyImage; combiner.SourceImage = image.Clone(); combiner.DestinationRectangle = rect300; var regionBounds = image.GetRegionBounds(null); combiner.SourcePoint = new LeadPoint(regionBounds.X, regionBounds.Y); //combiner.Flags = CombineCommandFlags.OperationAdd | CombineCommandFlags.Destination0 | CombineCommandFlags.Source1 | CombineCommandFlags.Destination0 ; combiner.Flags = CombineCommandFlags.OperationOr | CombineCommandFlags.Destination0;; // |CombineFastCommandFlags.OperationAverage; combiner.Run(fieldsOnlyImage); //combined = true; } catch (Exception exCombine) { logger.Error(exCombine, $"error combining field {field.Name} {rect300}"); } } var imageInfo = new ImageInfo() { Image = image, ImageFileInfo = new FileInfo(fileName) }; imageField.ImageInfo = imageInfo; if (!isBlock && zoneType != OcrZoneType.Graphic) { using (IOcrPage ocrPage = OcrEngine.CreatePage(image, OcrImageSharingMode.AutoDispose)) { OcrZone ocrZone = new OcrZone { ZoneType = zoneType, Bounds = new LeadRect(fudge, fudge, image.ImageSize.Width - fudge, image.ImageSize.Height - fudge) }; ocrPage.Zones.Add(ocrZone); ocrPage.Recognize(null); if (zoneType == OcrZoneType.Omr) { if (field.Name.Contains("C2NGVD1929")) { logger.Info(ocrZone.Bounds); } GetOmrReading(ocrPage, field, imageField); } else if (zoneType == OcrZoneType.Text) { var resultsPage = GetPageConfidence(ocrPage); imageField.FieldResult.Confidence = resultsPage.Confidence; char[] crlf = { '\r', '\n' }; imageField.FieldResult.Text = ocrPage.GetText(0).TrimEnd(crlf); } } } logger.Info( $"field {field.Name} {rect300} [{imageField.FieldResult.Text}] confidence: {imageField.FieldResult.Confidence}"); } catch (Exception exField) { logger.Error(exField, $"Error processing {field.Name}"); formResults.FieldsWithError++; imageField.FieldResult.Error = exField.Message; } newForm.ImageFields.Add(imageField); formResults.OcrFields.Add(imageField.FieldResult); formResults.Status = "FormMatched"; } catch (Exception ex) { logger.Error(ex, $"Error on field {field.Name} {rect300}"); newForm.Status = $"Error|Field {field.Name} {rect300}: [{ex.Message}]"; } } RasterCodecs.Save(PrepareOmrImage(fieldsOnlyImage), fileNameFieldOnly, RasterImageFormat.Jpeg, bitsPerPixel: 8); var googleResults = GoogleOcr(fileNameFieldOnly); if (googleResults.Count > 0) { var json = JsonConvert.SerializeObject(googleResults, Formatting.Indented); File.WriteAllText(googleResultsFile, json); MergeGoogleOcr(newForm, googleResults); } usedMasters.Add(currentMasterBlockForm); } else { newForm.Status = "Unmatched|No MasterForm match"; } logger.Info($"FilledForm processed {newForm.Name} {newForm.Status} {stopWatch.ElapsedMilliseconds} "); if (usedMasters.Count == BlockMasterForms.Count) { logger.Info("found all master forms"); break; } } stopWatch.Stop(); return(retForms); } catch (Exception ex) { logger.Error(ex, "Untrapped error found"); return(null); } }