static void Main(string[] args)
{
const string basePath = "..\\..\\..\\..\\images";
ZsImage srcImage = GetArgbImage(basePath, "t009.jpg");
ZsImage destImage = srcImage.Clone();
var srcMarkup = GetArea2D(basePath, "m009.png");
var destMarkup = srcMarkup.Translate(-300, -30);
destImage.CopyFromImage(destMarkup, destImage, srcMarkup);
destImage.FromArgbToBitmap()
.SaveTo("..\\..\\..\\target.png", ImageFormat.Png);
// Prepage setting for the PM algorithm
const byte patchSize = 5;
var settings = new PatchMatchSettings {
PatchSize = patchSize
};
// Init an nnf
var nnf = new Nnf(destImage.Width, destImage.Height, srcImage.Width, srcImage.Height, patchSize);
srcImage.FromArgbToRgb(new[] { 1.0, 1.0, 1.0 })
.FromRgbToLab();
destImage.FromArgbToRgb(new[] { 1.0, 1.0, 1.0 })
.FromRgbToLab();
destImage
.Clone()
.FromLabToRgb()
.FromRgbToBitmap()
.SaveTo($"..\\..\\..\\dest.png", ImageFormat.Png);
var patchMatchNnfBuilder = new PatchMatchNnfBuilder();
patchMatchNnfBuilder.RunRandomNnfInitIteration(nnf, destImage, srcImage, settings);
for (int j = 0; j < 3; j++)
{
patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Forward, settings);
Console.WriteLine($"\tIteration {j * 2}");
patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Backward, settings);
Console.WriteLine($"\tIteration {j * 2 + 1}");
}
nnf.ToRgbImage()
.FromRgbToBitmap()
.SaveTo($"..\\..\\..\\nnf.png", ImageFormat.Png);
nnf.RestoreImage(srcImage, 3, patchSize)
.FromLabToRgb()
.FromRgbToBitmap()
.SaveTo($"..\\..\\..\\restored.png", ImageFormat.Png);
}
public static async Task SaveImageLabToBlob(ZsImage imageLab, CloudBlobContainer container, string fileName)
{
var argbImage = imageLab
.Clone()
.FromLabToRgb()
.FromRgbToArgb(Area2D.Create(0, 0, imageLab.Width, imageLab.Height));
using (var bitmap = argbImage.FromArgbToBitmap())
using (var outputStream = new MemoryStream())
{
// modify image
bitmap.Save(outputStream, ImageFormat.Png);
// save the result back
outputStream.Position = 0;
var resultImageBlob = container.GetBlockBlobReference(fileName);
await resultImageBlob.UploadFromStreamAsync(outputStream);
}
}
/// <summary>
/// Build pyramids by downscaling the image and the markup.
/// We also apply a smoothing filter to the scaled images
/// to reduce high spatial ferquency introduced by scaling
/// (the filter is not applied to the inoainted area to avoid
/// inpainted object propagation out of its boundaries)
/// </summary>
/// <param name="levelsAmount">The levels amount.</param>
/// <param name="patchSize">Size of the patch.</param>
/// <returns></returns>
/// <exception cref="ArgumentOutOfRangeException">levelsAmount</exception>
/// <exception cref="Zavolokas.ImageProcessing.Inpainting.InitializationException"></exception>
/// <exception cref="Zavolokas.ImageProcessing.Inpainting.WrongImageSizeException"></exception>
/// <exception cref="Zavolokas.ImageProcessing.Inpainting.NoAreaToInpaintException"></exception>
/// <exception cref="AreaRemovedException"></exception>
public Pyramid Build(byte levelsAmount, byte patchSize = 11)
{
if (levelsAmount < 1)
{
throw new ArgumentOutOfRangeException(nameof(levelsAmount));
}
if (_imageArgb == null)
{
throw new InitializationException();
}
int w = _imageArgb.Width;
int h = _imageArgb.Height;
if (w == 1 || h == 1)
{
throw new WrongImageSizeException();
}
for (int levelInadex = 1; levelInadex < levelsAmount; levelInadex++)
{
if (w % 2 > 0 || h % 2 > 0)
{
throw new WrongImageSizeException();
}
w /= 2;
h /= 2;
if (w == 1 || h == 1)
{
throw new WrongImageSizeException();
}
}
var imageArea = Area2D.Create(0, 0, _imageArgb.Width, _imageArgb.Height);
if (_inpaintMarkup.Width != _imageArgb.Width || _inpaintMarkup.Height != _imageArgb.Height)
{
// Adjus inpaint markup image so that it become the same size as
// the image is
_inpaintMarkup = AlignImage(_inpaintMarkup, _imageArgb);
}
for (int i = 0; i < _donors.Count; i++)
{
if (_donors[i].Width != _imageArgb.Width || (_donors[i].Height != _imageArgb.Height))
{
_donors[i] = AlignImage(_donors[i], _imageArgb);
}
}
var inpaintArea = _inpaintMarkup.FromArgbToArea2D();
if (inpaintArea.IsEmpty)
{
throw new NoAreaToInpaintException();
}
if (imageArea.Substract(inpaintArea).IsEmpty)
{
throw new AreaRemovedException();
}
// Build pyramids
var images = new List <ZsImage>(levelsAmount);
var mappings = new List <Area2DMap>(levelsAmount);
var markups = new List <Area2D>(levelsAmount);
var mapBuilder = new InpaintMapBuilder(new Area2DMapBuilder());
for (byte levelIndex = 0; levelIndex < levelsAmount; levelIndex++)
{
// convert image to Lab color space and store it
var imageCopy = _imageArgb.Clone()
.FromArgbToRgb(_backgroundColor)
.FromRgbToLab();
images.Add(imageCopy);
inpaintArea = _inpaintMarkup.FromArgbToArea2D();
imageArea = Area2D.Create(0, 0, _imageArgb.Width, _imageArgb.Height);
var nnfSourceArea = imageArea.Substract(inpaintArea);
Area2D nnfTargetArea;
// Obtain target area for the NNF building based on the level
if (levelIndex == levelsAmount - 1)
{
// For the top level, the target area should be the whole image area
nnfTargetArea = imageArea;
}
else
{
// For the rest levels, the target area should be
// slightly bigger then the inpaint area.
nnfTargetArea = inpaintArea
.Dilation(patchSize)
.Intersect(imageArea);
}
// Create a mapping for the level.
mapBuilder.InitNewMap(imageArea);
foreach (var donor in _donors)
{
var donorArea = donor.FromArgbToArea2D();
if (!donorArea.IsEmpty)
{
donorArea = donorArea.Dilation(patchSize / 2); // This is very questionable dilation
mapBuilder.AddDonor(donorArea);
}
}
var mapping = mapBuilder
.SetInpaintArea(inpaintArea)
.ReduceDestArea(nnfTargetArea)
.Build();
mappings.Add(mapping);
markups.Add(inpaintArea);
if (levelIndex < levelsAmount - 1)
{
// downscale for the next level
// NOTE: we shouldn't blur out the inpainted area so it is not getting bigger!!
_imageArgb.PyramidDownArgb(nnfSourceArea);
_inpaintMarkup.PyramidDownArgb(false);
for (int i = 0; i < _donors.Count; i++)
{
_donors[i].PyramidDownArgb(false);
}
}
}
return(new Pyramid(images, markups, mappings));
}
public ZsImage Inpaint(ZsImage imageArgb, ZsImage markupArgb, InpaintSettings settings, IEnumerable <ZsImage> donorsArgb = null)
{
#region validate the inputs
if (imageArgb == null)
{
throw new ArgumentNullException(nameof(imageArgb));
}
if (imageArgb.NumberOfComponents != 4)
{
throw new BadImageFormatException($"{nameof(imageArgb)} is expected to be in ARGB format");
}
if (markupArgb == null)
{
throw new ArgumentNullException(nameof(markupArgb));
}
if (markupArgb.NumberOfComponents != 4)
{
throw new BadImageFormatException($"{nameof(markupArgb)} is expected to be in ARGB format");
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (donorsArgb == null)
{
donorsArgb = new ZsImage[0];
}
else if (donorsArgb.Any(donorArgb => donorArgb.NumberOfComponents != 4))
{
throw new BadImageFormatException($"{nameof(donorsArgb)} are expected to be in ARGB format");
}
#endregion
// Prepare input data
var originalImageArea = Area2D.Create(0, 0, imageArgb.Width, imageArgb.Height);
ZsImage imageToPorcess;
Pyramid imagePyramid;
var levelsAmount = _levelDetector.CalculateLevelsAmount(imageArgb, markupArgb, settings.PatchSize);
// Get the area of the image that can be scaled down required amount of times (levels)
var areaToProcess = GetImageAreaToProcess(imageArgb, markupArgb, levelsAmount);
if (!areaToProcess.IsSameAs(originalImageArea))
{
// We need to crop the original image, markup and donors
// since the area to process differs from the image area
imageToPorcess = CopyImageArea(imageArgb, areaToProcess);
var markup = CopyImageArea(markupArgb, areaToProcess);
var donors = donorsArgb.Select(donorImage => CopyImageArea(donorImage, areaToProcess));
imagePyramid = BuildPyramid(imageToPorcess, markup, donors, levelsAmount, settings.PatchSize);
}
else
{
imageToPorcess = imageArgb.Clone();
imagePyramid = BuildPyramid(imageToPorcess, markupArgb, donorsArgb, levelsAmount, settings.PatchSize);
}
imageToPorcess = InternalInpaint(imagePyramid, settings);
// paste result in the original bitmap where it was extracted from
var imageArea = Area2D.Create(0, 0, imageToPorcess.Width, imageToPorcess.Height);
imageToPorcess.FromLabToRgb()
.FromRgbToArgb(imageArea);
imageArgb.CopyFromImage(areaToProcess, imageToPorcess, imageArea);
return(imageArgb);
}
private ZsImage InternalInpaint(Pyramid pyramid, InpaintSettings settings)
{
#region cache settings
var patchSize = settings.PatchSize;
var calculator = settings.PatchDistanceCalculator;
var nnfSettings = settings.PatchMatch;
var changedPixelsPercentTreshold = settings.ChangedPixelsPercentTreshold;
var maxInpaintIterationsAmount = settings.MaxInpaintIterations;
var kStep = settings.MeanShift.KDecreaseStep;
var minK = settings.MeanShift.MinK;
#endregion
ZsImage image = null;
// go thru all the pyramid levels starting from the top one
Nnf nnf = null;
for (byte levelIndex = 0; levelIndex < pyramid.LevelsAmount; levelIndex++)
{
image = pyramid.GetImage(levelIndex);
var mapping = pyramid.GetMapping(levelIndex);
var inpaintArea = pyramid.GetInpaintArea(levelIndex);
var imageArea = Area2D.Create(0, 0, image.Width, image.Height);
// if there is a NNF built on the prev level
// scale it up
nnf = nnf == null
? new Nnf(image.Width, image.Height, image.Width, image.Height, patchSize)
: nnf.CloneAndScale2XWithUpdate(image, image, nnfSettings, mapping, calculator);
// start inpaint iterations
var k = settings.MeanShift.K;
for (var inpaintIterationIndex = 0; inpaintIterationIndex < maxInpaintIterationsAmount; inpaintIterationIndex++)
{
// Obtain pixels area.
// Pixels area defines which pixels are allowed to be used
// for the patches distance calculation. We must avoid pixels
// that we want to inpaint. That is why before the area is not
// inpainted - we should exclude this area.
var pixelsArea = imageArea;
if (levelIndex == 0 && inpaintIterationIndex == 0 && settings.IgnoreInpaintedPixelsOnFirstIteration)
{
pixelsArea = imageArea.Substract(inpaintArea);
}
// skip building NNF for the first iteration in the level
// unless it is top level (for the top one we haven't built NNF yet)
if (levelIndex == 0 || inpaintIterationIndex > 0)
{
// in order to find best matches for the inpainted area,
// we build NNF for this image as a dest and a source
// but excluding the inpainted area from the source area
// (our mapping already takes care of it)
_nnfBuilder.RunRandomNnfInitIteration(nnf, image, image, nnfSettings, calculator, mapping, pixelsArea);
_nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Forward, nnfSettings, calculator, mapping, pixelsArea);
_nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Backward, nnfSettings, calculator, mapping, pixelsArea);
_nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Forward, nnfSettings, calculator, mapping, pixelsArea);
_nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Backward, nnfSettings, calculator, mapping, pixelsArea);
_nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Forward, nnfSettings, calculator, mapping, pixelsArea);
}
var nnfNormalized = nnf.Clone();
nnfNormalized.Normalize();
// after we have the NNF - we calculate the values of the pixels in the inpainted area
var inpaintResult = Inpaint(image, inpaintArea, nnfNormalized, k, settings);
k = k > minK ? k - kStep : k;
if (IterationFinished != null)
{
var eventArgs = new InpaintIterationFinishedEventArgs
{
InpaintedLabImage = image.Clone(),
InpaintResult = inpaintResult,
LevelIndex = levelIndex,
InpaintIteration = inpaintIterationIndex
};
IterationFinished(this, eventArgs);
}
// if the change is smaller then a treshold, we quit
if (inpaintResult.ChangedPixelsPercent < changedPixelsPercentTreshold)
{
break;
}
//if (levelIndex == pyramid.LevelsAmount - 1) break;
}
}
return(image);
}
