public ZsImage Inpaint(ZsImage imageArgb, ZsImage markupArgb, InpaintSettings settings, IEnumerable <ZsImage> donorsArgb = null)
{
#region validate the inputs
if (imageArgb == null)
{
throw new ArgumentNullException(nameof(imageArgb));
}
if (imageArgb.NumberOfComponents != 4)
{
throw new BadImageFormatException($"{nameof(imageArgb)} is expected to be in ARGB format");
}
if (markupArgb == null)
{
throw new ArgumentNullException(nameof(markupArgb));
}
if (markupArgb.NumberOfComponents != 4)
{
throw new BadImageFormatException($"{nameof(markupArgb)} is expected to be in ARGB format");
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (donorsArgb == null)
{
donorsArgb = new ZsImage[0];
}
else if (donorsArgb.Any(donorArgb => donorArgb.NumberOfComponents != 4))
{
throw new BadImageFormatException($"{nameof(donorsArgb)} are expected to be in ARGB format");
}
#endregion
// Prepare input data
var originalImageArea = Area2D.Create(0, 0, imageArgb.Width, imageArgb.Height);
ZsImage imageToPorcess;
Pyramid imagePyramid;
var levelsAmount = _levelDetector.CalculateLevelsAmount(imageArgb, markupArgb, settings.PatchSize);
// Get the area of the image that can be scaled down required amount of times (levels)
var areaToProcess = GetImageAreaToProcess(imageArgb, markupArgb, levelsAmount);
if (!areaToProcess.IsSameAs(originalImageArea))
{
// We need to crop the original image, markup and donors
// since the area to process differs from the image area
imageToPorcess = CopyImageArea(imageArgb, areaToProcess);
var markup = CopyImageArea(markupArgb, areaToProcess);
var donors = donorsArgb.Select(donorImage => CopyImageArea(donorImage, areaToProcess));
imagePyramid = BuildPyramid(imageToPorcess, markup, donors, levelsAmount, settings.PatchSize);
}
else
{
imageToPorcess = imageArgb.Clone();
imagePyramid = BuildPyramid(imageToPorcess, markupArgb, donorsArgb, levelsAmount, settings.PatchSize);
}
imageToPorcess = InternalInpaint(imagePyramid, settings);
// paste result in the original bitmap where it was extracted from
var imageArea = Area2D.Create(0, 0, imageToPorcess.Width, imageToPorcess.Height);
imageToPorcess.FromLabToRgb()
.FromRgbToArgb(imageArea);
imageArgb.CopyFromImage(areaToProcess, imageToPorcess, imageArea);
return(imageArgb);
}
private ZsImage InternalInpaint(Pyramid pyramid, InpaintSettings settings)
{
#region cache settings
var patchSize = settings.PatchSize;
var calculator = settings.PatchDistanceCalculator;
var nnfSettings = settings.PatchMatch;
var changedPixelsPercentTreshold = settings.ChangedPixelsPercentTreshold;
var maxInpaintIterationsAmount = settings.MaxInpaintIterations;
var kStep = settings.MeanShift.KDecreaseStep;
var minK = settings.MeanShift.MinK;
#endregion
ZsImage image = null;
// go thru all the pyramid levels starting from the top one
Nnf nnf = null;
for (byte levelIndex = 0; levelIndex < pyramid.LevelsAmount; levelIndex++)
{
image = pyramid.GetImage(levelIndex);
var mapping = pyramid.GetMapping(levelIndex);
var inpaintArea = pyramid.GetInpaintArea(levelIndex);
var imageArea = Area2D.Create(0, 0, image.Width, image.Height);
// if there is a NNF built on the prev level
// scale it up
nnf = nnf == null
? new Nnf(image.Width, image.Height, image.Width, image.Height, patchSize)
: nnf.CloneAndScale2XWithUpdate(image, image, nnfSettings, mapping, calculator);
// start inpaint iterations
var k = settings.MeanShift.K;
for (var inpaintIterationIndex = 0; inpaintIterationIndex < maxInpaintIterationsAmount; inpaintIterationIndex++)
{
// Obtain pixels area.
// Pixels area defines which pixels are allowed to be used
// for the patches distance calculation. We must avoid pixels
// that we want to inpaint. That is why before the area is not
// inpainted - we should exclude this area.
var pixelsArea = imageArea;
if (levelIndex == 0 && inpaintIterationIndex == 0 && settings.IgnoreInpaintedPixelsOnFirstIteration)
{
pixelsArea = imageArea.Substract(inpaintArea);
}
// skip building NNF for the first iteration in the level
// unless it is top level (for the top one we haven't built NNF yet)
if (levelIndex == 0 || inpaintIterationIndex > 0)
{
// in order to find best matches for the inpainted area,
// we build NNF for this image as a dest and a source
// but excluding the inpainted area from the source area
// (our mapping already takes care of it)
_nnfBuilder.RunRandomNnfInitIteration(nnf, image, image, nnfSettings, calculator, mapping, pixelsArea);
_nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Forward, nnfSettings, calculator, mapping, pixelsArea);
_nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Backward, nnfSettings, calculator, mapping, pixelsArea);
_nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Forward, nnfSettings, calculator, mapping, pixelsArea);
_nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Backward, nnfSettings, calculator, mapping, pixelsArea);
_nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Forward, nnfSettings, calculator, mapping, pixelsArea);
}
var nnfNormalized = nnf.Clone();
nnfNormalized.Normalize();
// after we have the NNF - we calculate the values of the pixels in the inpainted area
var inpaintResult = Inpaint(image, inpaintArea, nnfNormalized, k, settings);
k = k > minK ? k - kStep : k;
if (IterationFinished != null)
{
var eventArgs = new InpaintIterationFinishedEventArgs
{
InpaintedLabImage = image.Clone(),
InpaintResult = inpaintResult,
LevelIndex = levelIndex,
InpaintIteration = inpaintIterationIndex
};
IterationFinished(this, eventArgs);
}
// if the change is smaller then a treshold, we quit
if (inpaintResult.ChangedPixelsPercent < changedPixelsPercentTreshold)
{
break;
}
//if (levelIndex == pyramid.LevelsAmount - 1) break;
}
}
return(image);
}