public ZsImage Inpaint(ZsImage imageArgb, ZsImage markupArgb, InpaintSettings settings, IEnumerable <ZsImage> donorsArgb = null) { #region validate the inputs if (imageArgb == null) { throw new ArgumentNullException(nameof(imageArgb)); } if (imageArgb.NumberOfComponents != 4) { throw new BadImageFormatException($"{nameof(imageArgb)} is expected to be in ARGB format"); } if (markupArgb == null) { throw new ArgumentNullException(nameof(markupArgb)); } if (markupArgb.NumberOfComponents != 4) { throw new BadImageFormatException($"{nameof(markupArgb)} is expected to be in ARGB format"); } if (settings == null) { throw new ArgumentNullException(nameof(settings)); } if (donorsArgb == null) { donorsArgb = new ZsImage[0]; } else if (donorsArgb.Any(donorArgb => donorArgb.NumberOfComponents != 4)) { throw new BadImageFormatException($"{nameof(donorsArgb)} are expected to be in ARGB format"); } #endregion // Prepare input data var originalImageArea = Area2D.Create(0, 0, imageArgb.Width, imageArgb.Height); ZsImage imageToPorcess; Pyramid imagePyramid; var levelsAmount = _levelDetector.CalculateLevelsAmount(imageArgb, markupArgb, settings.PatchSize); // Get the area of the image that can be scaled down required amount of times (levels) var areaToProcess = GetImageAreaToProcess(imageArgb, markupArgb, levelsAmount); if (!areaToProcess.IsSameAs(originalImageArea)) { // We need to crop the original image, markup and donors // since the area to process differs from the image area imageToPorcess = CopyImageArea(imageArgb, areaToProcess); var markup = CopyImageArea(markupArgb, areaToProcess); var donors = donorsArgb.Select(donorImage => CopyImageArea(donorImage, areaToProcess)); imagePyramid = BuildPyramid(imageToPorcess, markup, donors, levelsAmount, settings.PatchSize); } else { imageToPorcess = imageArgb.Clone(); imagePyramid = BuildPyramid(imageToPorcess, markupArgb, donorsArgb, levelsAmount, settings.PatchSize); } imageToPorcess = InternalInpaint(imagePyramid, settings); // paste result in the original bitmap where it was extracted from var imageArea = Area2D.Create(0, 0, imageToPorcess.Width, imageToPorcess.Height); imageToPorcess.FromLabToRgb() .FromRgbToArgb(imageArea); imageArgb.CopyFromImage(areaToProcess, imageToPorcess, imageArea); return(imageArgb); }
private ZsImage InternalInpaint(Pyramid pyramid, InpaintSettings settings) { #region cache settings var patchSize = settings.PatchSize; var calculator = settings.PatchDistanceCalculator; var nnfSettings = settings.PatchMatch; var changedPixelsPercentTreshold = settings.ChangedPixelsPercentTreshold; var maxInpaintIterationsAmount = settings.MaxInpaintIterations; var kStep = settings.MeanShift.KDecreaseStep; var minK = settings.MeanShift.MinK; #endregion ZsImage image = null; // go thru all the pyramid levels starting from the top one Nnf nnf = null; for (byte levelIndex = 0; levelIndex < pyramid.LevelsAmount; levelIndex++) { image = pyramid.GetImage(levelIndex); var mapping = pyramid.GetMapping(levelIndex); var inpaintArea = pyramid.GetInpaintArea(levelIndex); var imageArea = Area2D.Create(0, 0, image.Width, image.Height); // if there is a NNF built on the prev level // scale it up nnf = nnf == null ? new Nnf(image.Width, image.Height, image.Width, image.Height, patchSize) : nnf.CloneAndScale2XWithUpdate(image, image, nnfSettings, mapping, calculator); // start inpaint iterations var k = settings.MeanShift.K; for (var inpaintIterationIndex = 0; inpaintIterationIndex < maxInpaintIterationsAmount; inpaintIterationIndex++) { // Obtain pixels area. // Pixels area defines which pixels are allowed to be used // for the patches distance calculation. We must avoid pixels // that we want to inpaint. That is why before the area is not // inpainted - we should exclude this area. var pixelsArea = imageArea; if (levelIndex == 0 && inpaintIterationIndex == 0 && settings.IgnoreInpaintedPixelsOnFirstIteration) { pixelsArea = imageArea.Substract(inpaintArea); } // skip building NNF for the first iteration in the level // unless it is top level (for the top one we haven't built NNF yet) if (levelIndex == 0 || inpaintIterationIndex > 0) { // in order to find best matches for the inpainted area, // we build NNF for this image as a dest and a source // but excluding the inpainted area from the source area // (our mapping already takes care of it) _nnfBuilder.RunRandomNnfInitIteration(nnf, image, image, nnfSettings, calculator, mapping, pixelsArea); _nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Forward, nnfSettings, calculator, mapping, pixelsArea); _nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Backward, nnfSettings, calculator, mapping, pixelsArea); _nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Forward, nnfSettings, calculator, mapping, pixelsArea); _nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Backward, nnfSettings, calculator, mapping, pixelsArea); _nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Forward, nnfSettings, calculator, mapping, pixelsArea); } var nnfNormalized = nnf.Clone(); nnfNormalized.Normalize(); // after we have the NNF - we calculate the values of the pixels in the inpainted area var inpaintResult = Inpaint(image, inpaintArea, nnfNormalized, k, settings); k = k > minK ? k - kStep : k; if (IterationFinished != null) { var eventArgs = new InpaintIterationFinishedEventArgs { InpaintedLabImage = image.Clone(), InpaintResult = inpaintResult, LevelIndex = levelIndex, InpaintIteration = inpaintIterationIndex }; IterationFinished(this, eventArgs); } // if the change is smaller then a treshold, we quit if (inpaintResult.ChangedPixelsPercent < changedPixelsPercentTreshold) { break; } //if (levelIndex == pyramid.LevelsAmount - 1) break; } } return(image); }