/// <summary> /// Runs the NNF build iteration using the whole areas of the dest and the source images. Uses CIE76 to calculate patch similarity. /// </summary> /// <param name="nnf">The NNF.</param> /// <param name="destImage">The dest image. For each patch at this image we will look for a similar one at the source image.</param> /// <param name="srcImage">The source image. Source of the patches for the dest image.</param> /// <param name="direction">The direction to look for a patches.</param> /// <param name="settings">The settings that control parameters of the algorithm.</param> public void RunBuildNnfIteration(Nnf nnf, ZsImage destImage, ZsImage srcImage, NeighboursCheckDirection direction, PatchMatchSettings settings) { var patchDistanceCalculator = ImagePatchDistance.Cie76; RunBuildNnfIteration(nnf, destImage, srcImage, direction, settings, patchDistanceCalculator); }
public static async Task <InpaintingResult> InpaintImage([ActivityTrigger] NnfInputData input) { var container = BlobHelper.OpenBlobContainer(input.Container); var imageBlob = container.GetBlockBlobReference(input.Image); var image = (await BlobHelper.ConvertBlobToArgbImage(imageBlob)) .FromArgbToRgb(new[] { 0.0, 0.0, 0.0 }) .FromRgbToLab(); var inpaintAreaState = BlobHelper.ReadFromBlob <Area2DState>(input.InpaintAreaName, container); var inpaintArea = Area2D.RestoreFrom(inpaintAreaState); var nnfState = BlobHelper.ReadFromBlob <NnfState>(input.NnfName, container); var nnf = new Nnf(nnfState); nnf.Normalize(); // after we have the NNF - we calculate the values of the pixels in the inpainted area var inpaintResult = Inpaint(image, inpaintArea, nnf, input.K, input.Settings); await BlobHelper.SaveImageLabToBlob(image, container, input.Image); // TODO: remove it later it is for debug purpose. await BlobHelper.SaveImageLabToBlob(image, container, $"{input.LevelIndex}_{input.IterationIndex}.png"); return(inpaintResult); }
public PmData(ZsImage destImage, ZsImage srcImage, Nnf nnf, Area2DMap map) { if (destImage == null) { throw new ArgumentNullException(nameof(destImage)); } if (srcImage == null) { throw new ArgumentNullException(nameof(srcImage)); } if (map == null) { throw new ArgumentNullException(nameof(map)); } if (nnf == null) { throw new ArgumentNullException(nameof(nnf)); } Map = map; Nnf = nnf; DestImage = destImage; SrcImage = srcImage; DestImagePixelsArea = Area2D.Create(0, 0, destImage.Width, destImage.Height); Settings = new PatchMatchSettings(); }
public PmData(ZsImage destImage, ZsImage srcImage) { if (destImage == null) { throw new ArgumentNullException(nameof(destImage)); } if (srcImage == null) { throw new ArgumentNullException(nameof(srcImage)); } DestImage = destImage; SrcImage = srcImage; Nnf = new Nnf(destImage.Width, destImage.Height, srcImage.Width, srcImage.Height); Settings = new PatchMatchSettings(); var destImageArea = Area2D.Create(0, 0, destImage.Width, destImage.Height); DestImagePixelsArea = destImageArea; var mapBuilder = new Area2DMapBuilder(); mapBuilder.InitNewMap( destImageArea, Area2D.Create(0, 0, srcImage.Width, srcImage.Height)); Map = mapBuilder.Build(); }
public Nnf Clone() { var clone = new Nnf(DstWidth, DstHeight, SourceWidth, SourceHeight, PatchSize); _nnf.CopyTo(clone._nnf, 0); return(clone); }
private static Area2DMap CreateMapping(Nnf nnf) { var dest = Area2D.Create(0, 0, nnf.DstWidth, nnf.DstHeight); var src = Area2D.Create(0, 0, nnf.SourceWidth, nnf.SourceHeight); return(new Area2DMapBuilder() .InitNewMap(dest, src) .Build()); }
static void Main(string[] args) { const string basePath = "..\\..\\..\\..\\images"; ZsImage srcImage = GetArgbImage(basePath, "t009.jpg"); ZsImage destImage = srcImage.Clone(); var srcMarkup = GetArea2D(basePath, "m009.png"); var destMarkup = srcMarkup.Translate(-300, -30); destImage.CopyFromImage(destMarkup, destImage, srcMarkup); destImage.FromArgbToBitmap() .SaveTo("..\\..\\..\\target.png", ImageFormat.Png); // Prepage setting for the PM algorithm const byte patchSize = 5; var settings = new PatchMatchSettings { PatchSize = patchSize }; // Init an nnf var nnf = new Nnf(destImage.Width, destImage.Height, srcImage.Width, srcImage.Height, patchSize); srcImage.FromArgbToRgb(new[] { 1.0, 1.0, 1.0 }) .FromRgbToLab(); destImage.FromArgbToRgb(new[] { 1.0, 1.0, 1.0 }) .FromRgbToLab(); destImage .Clone() .FromLabToRgb() .FromRgbToBitmap() .SaveTo($"..\\..\\..\\dest.png", ImageFormat.Png); var patchMatchNnfBuilder = new PatchMatchNnfBuilder(); patchMatchNnfBuilder.RunRandomNnfInitIteration(nnf, destImage, srcImage, settings); for (int j = 0; j < 3; j++) { patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Forward, settings); Console.WriteLine($"\tIteration {j * 2}"); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Backward, settings); Console.WriteLine($"\tIteration {j * 2 + 1}"); } nnf.ToRgbImage() .FromRgbToBitmap() .SaveTo($"..\\..\\..\\nnf.png", ImageFormat.Png); nnf.RestoreImage(srcImage, 3, patchSize) .FromLabToRgb() .FromRgbToBitmap() .SaveTo($"..\\..\\..\\restored.png", ImageFormat.Png); }
static void Main(string[] args) { const string basePath = "..\\..\\..\\images"; var destImageName = "pm2small.png"; var srcImageName = "pm1small.png"; var emptyAreaImageName = "pm2small_ignore.png"; // This is our input data. var destImage = GetLabImage(basePath, destImageName); var srcImage = GetLabImage(basePath, srcImageName); var destArea = Area2D.Create(0, 0, destImage.Width, destImage.Height); var srcArea = Area2D.Create(0, 0, srcImage.Width, srcImage.Height); var emptyArea = GetArea2D(basePath, emptyAreaImageName); var destPixelsArea = destArea.Substract(emptyArea); var map = new Area2DMapBuilder() .InitNewMap(destArea, srcArea) .Build(); const byte patchSize = 5; var settings = new PatchMatchSettings { PatchSize = patchSize }; var nnf = new Nnf(destImage.Width, destImage.Height, srcImage.Width, srcImage.Height, patchSize); var calculator = ImagePatchDistance.Cie76; var patchMatchNnfBuilder = new PatchMatchNnfBuilder(); // Create the nnf the images // with a couple of iterations. patchMatchNnfBuilder.RunRandomNnfInitIteration(nnf, destImage, srcImage, settings, calculator, map, destPixelsArea); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map, destPixelsArea); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Backward, settings, calculator, map, destPixelsArea); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map, destPixelsArea); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Backward, settings, calculator, map, destPixelsArea); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map, destPixelsArea); // Restore dest image from the NNF and source image. nnf .RestoreImage(srcImage, 3, settings.PatchSize) .FromLabToRgb() .FromRgbToBitmap() .SaveTo(@"..\..\restored.png", ImageFormat.Png); // Convert the NNF to an image, save and show it nnf .ToRgbImage() .FromRgbToBitmap() .SaveTo(@"..\..\nnf.png", ImageFormat.Png) .ShowFile(); Console.WriteLine($"PatchMatchPipeline processing is finished."); }
public static async Task CreateNnf([ActivityTrigger] NnfInputData input) { var container = BlobHelper.OpenBlobContainer(input.Container); var imageBlob = container.GetBlockBlobReference(input.Image); var imageArgb = await BlobHelper.ConvertBlobToArgbImage(imageBlob); var nnf = new Nnf(imageArgb.Width, imageArgb.Height, imageArgb.Width, imageArgb.Height, input.Settings.PatchSize); var nnfData = JsonConvert.SerializeObject(nnf.GetState()); BlobHelper.SaveJsonToBlob(nnfData, container, input.NnfName); }
/// <summary> /// Normalizes the specified NNF. /// </summary> /// <param name="nnf">The NNF.</param> /// <exception cref="ArgumentNullException">nnf</exception> public static void Normalize(this Nnf nnf) { if (nnf == null) { throw new ArgumentNullException(nameof(nnf)); } var destArea = Area2D.Create(0, 0, nnf.DstWidth, nnf.DstHeight); nnf.Normalize(destArea); }
/// <summary> /// Normalizes the NNF in the specified dest area. /// </summary> /// <param name="nnf">The NNF.</param> /// <param name="destArea">The dest area.</param> /// <exception cref="ArgumentNullException"> /// nnf /// or /// destArea /// </exception> public static void Normalize(this Nnf nnf, Area2D destArea) { if (nnf == null) { throw new ArgumentNullException(nameof(nnf)); } if (destArea == null) { throw new ArgumentNullException(nameof(destArea)); } var nnfdata = nnf.GetNnfItems(); checked { var distances = new double[destArea.ElementsCount]; var pointIndexes = new int[destArea.ElementsCount]; destArea.FillMappedPointsIndexes(pointIndexes, nnf.DstWidth); var dinstancesSum = 0.0; for (int destPointIndex = 0; destPointIndex < distances.Length; destPointIndex++) { var nnfPos = pointIndexes[destPointIndex] * 2; double distance = nnfdata[nnfPos + 1]; distances[destPointIndex] = distance; dinstancesSum += distance; } double mean = dinstancesSum / distances.Length; var squareDistances = new double[distances.Length]; double squreDistancesSum = 0.0; for (int i = 0; i < distances.Length; i++) { var distToMean = distances[i] - mean; double distToMeanCube = distToMean * distToMean; squareDistances[i] = distToMeanCube; squreDistancesSum += distToMeanCube; } double sigma = System.Math.Sqrt(squreDistancesSum / (squareDistances.Length - 1)); for (int destPointIndex = 0; destPointIndex < distances.Length; destPointIndex++) { var nnfPos = pointIndexes[destPointIndex] * 2; var dist = distances[destPointIndex]; dist = (dist - mean) / sigma; if (dist < 0) { dist = -dist; } nnfdata[nnfPos + 1] = dist; } } }
/// <summary> /// Runs the random NNF initialization iteration for the associated areas of the dest and the source images. /// </summary> /// <param name="nnf">The NNF.</param> /// <param name="destImage">The dest image. For each patch at this image we will look for a similar one at the source image.</param> /// <param name="srcImage">The source image. Source of the patches for the dest image.</param> /// <param name="settings">The settings that control parameters of the algorithm.</param> /// <param name="patchDistanceCalculator">The calculator that calculates similarity of two patches. By deafult the Cie76 is used.</param> /// <param name="areasMapping">The areas mapping. By default whole area of the dest image is associated with the whole area of the source image.</param> /// <exception cref="ArgumentNullException">destImage</exception> public void RunRandomNnfInitIteration(Nnf nnf, ZsImage destImage, ZsImage srcImage, PatchMatchSettings settings, ImagePatchDistanceCalculator patchDistanceCalculator, Area2DMap areasMapping) { if (destImage == null) { throw new ArgumentNullException(nameof(destImage)); } var destPixelsArea = Area2D.Create(0, 0, destImage.Width, destImage.Height); RunRandomNnfInitIteration(nnf, destImage, srcImage, settings, patchDistanceCalculator, areasMapping, destPixelsArea); }
public static async Task NnfInitIteration([ActivityTrigger] NnfInputData input) { var container = BlobHelper.OpenBlobContainer(input.Container); var imageBlob = container.GetBlockBlobReference(input.Image); var imageArgb = await BlobHelper.ConvertBlobToArgbImage(imageBlob); var image = imageArgb .FromArgbToRgb(new[] { 0.0, 0.0, 0.0 }) .FromRgbToLab(); var imageArea = Area2D.Create(0, 0, image.Width, image.Height); var pixelsArea = imageArea; var nnfSettings = input.Settings.PatchMatch; var calculator = input.IsCie79Calc ? ImagePatchDistance.Cie76 : ImagePatchDistance.Cie2000; var nnfState = BlobHelper.ReadFromBlob <NnfState>(input.NnfName, container); var nnf = new Nnf(nnfState); var mappingState = BlobHelper.ReadFromBlob <Area2DMapState>(input.Mapping, container); var mapping = new Area2DMap(mappingState); if (input.ExcludeInpaintArea) { var inpaintAreaState = BlobHelper.ReadFromBlob <Area2DState>(input.InpaintAreaName, container); var inpaintArea = Area2D.RestoreFrom(inpaintAreaState); pixelsArea = imageArea.Substract(inpaintArea); } var nnfBuilder = new PatchMatchNnfBuilder(); nnfBuilder.RunRandomNnfInitIteration(nnf, image, image, nnfSettings, calculator, mapping, pixelsArea); var nnfData = JsonConvert.SerializeObject(nnf.GetState()); BlobHelper.SaveJsonToBlob(nnfData, container, input.NnfName); foreach (var subNnfName in input.SplittedNnfNames) { BlobHelper.SaveJsonToBlob(nnfData, container, subNnfName); } }
/// <summary> /// Runs the random NNF initialization iteration using the whole areas of the dest and the source images. /// </summary> /// <param name="nnf">The NNF.</param> /// <param name="destImage">The dest image. For each patch at this image we will look for a similar one at the source image.</param> /// <param name="srcImage">The source image. Source of the patches for the dest image.</param> /// <param name="settings">The settings that control parameters of the algorithm.</param> /// <param name="patchDistanceCalculator">The calculator that calculates similarity of two patches. By deafult the Cie76 is used.</param> /// <exception cref="ArgumentNullException"> /// destImage /// or /// srcImage /// </exception> public void RunRandomNnfInitIteration(Nnf nnf, ZsImage destImage, ZsImage srcImage, PatchMatchSettings settings, ImagePatchDistanceCalculator patchDistanceCalculator) { if (destImage == null) { throw new ArgumentNullException(nameof(destImage)); } if (srcImage == null) { throw new ArgumentNullException(nameof(srcImage)); } var destArea = Area2D.Create(0, 0, destImage.Width, destImage.Height); var srcArea = Area2D.Create(0, 0, srcImage.Width, srcImage.Height); var map = new Area2DMapBuilder() .InitNewMap(destArea, srcArea) .Build(); RunRandomNnfInitIteration(nnf, destImage, srcImage, settings, patchDistanceCalculator, map); }
/// <summary> /// Clones the NNF and scales it up 2 times with distances recalculation. /// </summary> /// <param name="nnf">The NNF.</param> /// <param name="scaledDestImage">The scaled dest image.</param> /// <param name="scaledSrcImage">The scaled source image.</param> /// <param name="options">The options for parallel processing.</param> /// <param name="patchDistanceCalculator">The calculator that calculates similarity of two patches. By deafult the Cie76 is used.</param> /// <param name="destPixelsArea">Area on the dest image that actually containes pixels. By default is the area of the entire image.</param> /// <returns></returns> /// <exception cref="ArgumentNullException"> /// nnf /// or /// scaledDestImage /// or /// scaledSrcImage /// </exception> public static Nnf CloneAndScale2XWithUpdate(this Nnf nnf, ZsImage scaledDestImage, ZsImage scaledSrcImage, ParallelOptions options = null, ImagePatchDistanceCalculator patchDistanceCalculator = null, Area2D destPixelsArea = null) { if (scaledDestImage == null) { throw new ArgumentNullException(nameof(scaledDestImage)); } if (scaledSrcImage == null) { throw new ArgumentNullException(nameof(scaledSrcImage)); } var destArea = Area2D.Create(0, 0, scaledDestImage.Width, scaledDestImage.Height); var srcArea = Area2D.Create(0, 0, scaledSrcImage.Width, scaledSrcImage.Height); Area2DMap scaledMap = new Area2DMapBuilder() .InitNewMap(destArea, srcArea) .Build(); return(nnf.CloneAndScale2XWithUpdate(scaledDestImage, scaledSrcImage, options, scaledMap, patchDistanceCalculator, destPixelsArea)); }
public static async Task ScaleNnf([ActivityTrigger] NnfInputData input) { var container = BlobHelper.OpenBlobContainer(input.Container); var imageBlob = container.GetBlockBlobReference(input.Image); var image = (await BlobHelper.ConvertBlobToArgbImage(imageBlob)) .FromArgbToRgb(new[] { 0.0, 0.0, 0.0 }) .FromRgbToLab(); var calculator = input.IsCie79Calc ? ImagePatchDistance.Cie76 : ImagePatchDistance.Cie2000; var mappingState = BlobHelper.ReadFromBlob <Area2DMapState>(input.Mapping, container); var mapping = new Area2DMap(mappingState); var nnfState = BlobHelper.ReadFromBlob <NnfState>($"nnf{input.LevelIndex - 1}.json", container); var nnf = new Nnf(nnfState); nnf = nnf.CloneAndScale2XWithUpdate(image, image, input.Settings.PatchMatch, mapping, calculator); var nnfData = JsonConvert.SerializeObject(nnf.GetState()); BlobHelper.SaveJsonToBlob(nnfData, container, input.NnfName); }
/// <summary> /// Clones the NNF and scales it up 2 times without distances recalculation. /// </summary> /// <param name="nnf">The NNF.</param> /// <param name="scaledDestImage">The scaled dest image.</param> /// <param name="scaledSrcImage">The scaled source image.</param> /// <param name="options">The options for parallel processing.</param> /// <param name="scaledMap">The areas mapping. By default whole area of the dest image is associated with the whole area of the source image.</param> /// <returns></returns> /// <exception cref="ArgumentNullException"> /// nnf /// or /// scaledDestImage /// or /// scaledSrcImage /// </exception> public static Nnf CloneAndScaleNnf2X(this Nnf nnf, ZsImage scaledDestImage, ZsImage scaledSrcImage, ParallelOptions options = null, Area2DMap scaledMap = null) { if (nnf == null) { throw new ArgumentNullException(nameof(nnf)); } if (scaledDestImage == null) { throw new ArgumentNullException(nameof(scaledDestImage)); } if (scaledSrcImage == null) { throw new ArgumentNullException(nameof(scaledSrcImage)); } if (scaledMap == null) { var destArea = Area2D.Create(0, 0, scaledDestImage.Width, scaledDestImage.Height); var srcArea = Area2D.Create(0, 0, scaledSrcImage.Width, scaledSrcImage.Height); scaledMap = new Area2DMapBuilder() .InitNewMap(destArea, srcArea) .Build(); } if (options == null) { options = new ParallelOptions(); } var destImageWidth = scaledDestImage.Width; var srcImageWidth = scaledSrcImage.Width; var sameSrcAndDest = scaledDestImage == scaledSrcImage; var mappings = scaledMap.ExtractMappedAreasInfo(destImageWidth, srcImageWidth, true); var nnf2x = new Nnf(nnf.DstWidth * 2, nnf.DstHeight * 2, nnf.SourceWidth * 2, nnf.SourceHeight * 2, nnf.PatchSize); var nnfDestWidth = nnf.DstWidth; var nnfSourceWidth = nnf.SourceWidth; var nnf2xSourceWidth = nnf2x.SourceWidth; var nnf2xDstWidth = nnf2x.DstWidth; var nnfData = nnf.GetNnfItems(); var nnf2xData = nnf2x.GetNnfItems(); // Decide on how many partitions we should divade the processing // of the elements. int nnfPointsAmount = nnf.DstWidth * nnf.DstHeight; var partsCount = nnfPointsAmount > options.NotDividableMinAmountElements ? options.ThreadsCount : 1; var partSize = nnfPointsAmount / partsCount; var offs = new[] { new[] { 0, 0 }, new[] { 1, 0 }, new[] { 0, 1 }, new[] { 1, 1 } }; Parallel.For(0, partsCount, partIndex => //for (int partIndex = 0; partIndex < partsCount; partIndex++) { bool isPatchFit = false; var mappedAreasInfos = new MappedAreasInfo[mappings.Length]; for (var i = 0; i < mappings.Length; i++) { mappedAreasInfos[i] = mappings[i].Clone(); } var firstPointIndex = partIndex * partSize; var lastPointIndex = firstPointIndex + partSize - 1; if (partIndex == partsCount - 1) { lastPointIndex = nnfPointsAmount - 1; } if (lastPointIndex > nnfPointsAmount) { lastPointIndex = nnfPointsAmount - 1; } MappedAreasInfo mappedAreasInfo = null; for (var j = firstPointIndex; j <= lastPointIndex; j++) { var destPx = j; var destX = destPx % nnfDestWidth; var destY = destPx / nnfDestWidth; // Find var srcPointIndex = nnfData[destY * nnfDestWidth * 2 + destX * 2]; // / 2; int srcY = (int)(srcPointIndex / nnfSourceWidth); // * 2; int srcX = (int)(srcPointIndex % nnfSourceWidth); // * 2; var dist = nnfData[destY * nnfDestWidth * 2 + destX * 2 + 1]; var nY = destY * 2; var nX = destX * 2; for (int i = 0; i < offs.Length; i++) { var destPointIndex = (nY + offs[i][1]) * nnf2xDstWidth + nX + offs[i][0]; mappedAreasInfo = mappedAreasInfos.FirstOrDefault(mai => mai.DestAreaPointsIndexesSet.Contains(destPointIndex)); if (mappedAreasInfo != null) { nnf2xData[destPointIndex * 2] = (srcY * 2 + offs[i][1]) * nnf2xSourceWidth + (srcX * 2 + offs[i][0]); nnf2xData[destPointIndex * 2 + 1] = dist; } else { if (sameSrcAndDest) { // when the source and the dest image is the same one, the best // corresponding patch is the patch itself! nnf2xData[destPointIndex * 2] = destPointIndex; nnf2xData[destPointIndex * 2 + 1] = 0; } else { nnf2xData[destPointIndex * 2] = (srcY * 2 + offs[i][1]) * nnf2xSourceWidth + (srcX * 2 + offs[i][0]); nnf2xData[destPointIndex * 2 + 1] = nnfData[destY * nnfDestWidth * 2 + destX * 2 + 1]; } } } } } ); return(nnf2x); }
/// <summary> /// Clones the NNF and scales it up 2 times with distances recalculation. /// </summary> /// <param name="nnf">The NNF.</param> /// <param name="scaledDestImage">The scaled dest image.</param> /// <param name="scaledSrcImage">The scaled source image.</param> /// <param name="options">The options for parallel processing.</param> /// <param name="scaledMap">The areas mapping. By default whole area of the dest image is associated with the whole area of the source image.</param> /// <param name="patchDistanceCalculator">The calculator that calculates similarity of two patches. By deafult the Cie76 is used.</param> /// <param name="destPixelsArea">Area on the dest image that actually containes pixels. By default is the area of the entire image.</param> /// <returns></returns> /// <exception cref="ArgumentNullException">nnf /// or /// scaledDestImage /// or /// scaledSrcImage /// or /// scaledMap</exception> public static unsafe Nnf CloneAndScale2XWithUpdate(this Nnf nnf, ZsImage scaledDestImage, ZsImage scaledSrcImage, ParallelOptions options, Area2DMap scaledMap, ImagePatchDistanceCalculator patchDistanceCalculator = null, Area2D destPixelsArea = null) { if (nnf == null) { throw new ArgumentNullException(nameof(nnf)); } if (scaledDestImage == null) { throw new ArgumentNullException(nameof(scaledDestImage)); } if (scaledSrcImage == null) { throw new ArgumentNullException(nameof(scaledSrcImage)); } if (scaledMap == null) { throw new ArgumentNullException(nameof(scaledMap)); } if (destPixelsArea == null) { destPixelsArea = Area2D.Create(0, 0, scaledDestImage.Width, scaledDestImage.Height); } if (patchDistanceCalculator == null) { patchDistanceCalculator = ImagePatchDistance.Cie76; } if (options == null) { options = new ParallelOptions(); } var patchSize = nnf.PatchSize; var patchLength = patchSize * patchSize; var destImageWidth = scaledDestImage.Width; var srcImageWidth = scaledSrcImage.Width; var sameSrcAndDest = scaledDestImage == scaledSrcImage; var pixelsArea = (scaledMap as IAreasMapping).DestArea; //var destPointIndexes = GetDestPointsIndexes(pixelsArea, destImageWidth, NeighboursCheckDirection.Forward); pixelsArea = pixelsArea.Intersect(destPixelsArea); var destAvailablePixelsIndexes = pixelsArea.GetPointsIndexes(destImageWidth); var mappings = scaledMap.ExtractMappedAreasInfo(destImageWidth, srcImageWidth, true); var nnf2x = new Nnf(nnf.DstWidth * 2, nnf.DstHeight * 2, nnf.SourceWidth * 2, nnf.SourceHeight * 2, nnf.PatchSize); var nnfDestWidth = nnf.DstWidth; var nnfSourceWidth = nnf.SourceWidth; var nnf2xSourceWidth = nnf2x.SourceWidth; var nnf2xDstWidth = nnf2x.DstWidth; var nnfData = nnf.GetNnfItems(); var nnf2xData = nnf2x.GetNnfItems(); // Decide on how many partitions we should divade the processing // of the elements. int nnfPointsAmount = nnf.DstWidth * nnf.DstHeight; var partsCount = nnfPointsAmount > options.NotDividableMinAmountElements ? options.ThreadsCount : 1; var partSize = nnfPointsAmount / partsCount; var offs = new[] { new[] { 0, 0 }, new[] { 1, 0 }, new[] { 0, 1 }, new[] { 1, 1 } }; Parallel.For(0, partsCount, partIndex => //for (int partIndex = 0; partIndex < partsCount; partIndex++) { bool isPatchFit = false; // Init the dest & source patch var destPatchPixelsIndexes = new int[patchLength]; var srcPatchPixelsIndexes = new int[patchLength]; //var destPointsIndexesSet = new HashSet<int>(destPointIndexes); var destAvailablePixelsIndexesSet = new HashSet <int>(destAvailablePixelsIndexes); var mappedAreasInfos = new MappedAreasInfo[mappings.Length]; for (var i = 0; i < mappings.Length; i++) { mappedAreasInfos[i] = mappings[i].Clone(); } var firstPointIndex = partIndex * partSize; var lastPointIndex = firstPointIndex + partSize - 1; if (partIndex == partsCount - 1) { lastPointIndex = nnfPointsAmount - 1; } if (lastPointIndex > nnfPointsAmount) { lastPointIndex = nnfPointsAmount - 1; } fixed(double *destImagePixelsDataP = scaledDestImage.PixelsData) fixed(double *sourceImagePixelsDataP = scaledSrcImage.PixelsData) fixed(int *srcPatchPixelsIndexesP = srcPatchPixelsIndexes) fixed(int *destPatchPixelsIndexesP = destPatchPixelsIndexes) { MappedAreasInfo mappedAreasInfo = null; for (var j = firstPointIndex; j <= lastPointIndex; j++) { var destPx = j; var destX = destPx % nnfDestWidth; var destY = destPx / nnfDestWidth; // Find var srcPointIndex = nnfData[destY * nnfDestWidth * 2 + destX * 2]; // / 2; int srcY = (int)(srcPointIndex / nnfSourceWidth); // * 2; int srcX = (int)(srcPointIndex % nnfSourceWidth); // * 2; var nY = destY * 2; var nX = destX * 2; for (int i = 0; i < offs.Length; i++) { var destPointIndex = (nY + offs[i][1]) * nnf2xDstWidth + nX + offs[i][0]; mappedAreasInfo = mappedAreasInfos.FirstOrDefault(mai => mai.DestAreaPointsIndexesSet.Contains(destPointIndex)); if (mappedAreasInfo != null) { nnf2xData[destPointIndex * 2] = (srcY * 2 + offs[i][1]) * nnf2xSourceWidth + (srcX * 2 + offs[i][0]); Utils.PopulatePatchPixelsIndexes(srcPatchPixelsIndexesP, srcX + offs[i][0], srcY + offs[i][1], patchSize, nnf2xSourceWidth, mappedAreasInfo.SrcAreaPointsIndexesSet, out isPatchFit); Utils.PopulatePatchPixelsIndexes(destPatchPixelsIndexesP, destX + offs[i][0], destY + offs[i][1], patchSize, destImageWidth, destAvailablePixelsIndexesSet, out isPatchFit); nnf2xData[destPointIndex * 2 + 1] = patchDistanceCalculator.Calculate(destPatchPixelsIndexesP, srcPatchPixelsIndexesP, double.MaxValue, destImagePixelsDataP, sourceImagePixelsDataP, scaledDestImage, scaledSrcImage, patchLength); } else { if (sameSrcAndDest) { // when the source and the dest image is the same one, the best // corresponding patch is the patch itself! nnf2xData[destPointIndex * 2] = destPointIndex; nnf2xData[destPointIndex * 2 + 1] = 0; } else { nnf2xData[destPointIndex * 2] = (srcY * 2 + offs[i][1]) * nnf2xSourceWidth + (srcX * 2 + offs[i][0]); nnf2xData[destPointIndex * 2 + 1] = nnfData[destY * nnfDestWidth * 2 + destX * 2 + 1]; } } } } } } ); return(nnf2x); }
static void Main(string[] args) { const string basePath = "..\\..\\..\\images"; const string srcImageName = "pm1.png"; const string destImageName = "pm2.png"; const int patchSize = 5; // Prepare 2 source images - one small and another 2x bigger var srcBitmap = new Bitmap(Path.Combine(basePath, srcImageName)); var srcImage = srcBitmap .ToRgbImage() .FromRgbToLab(); var srcSmallBitmap = srcBitmap.CloneWithScaleTo(srcBitmap.Width / 2, srcBitmap.Height / 2); var srcSmallImage = srcSmallBitmap .ToRgbImage() .FromRgbToLab(); srcSmallBitmap.Dispose(); srcBitmap.Dispose(); var destBitmap = new Bitmap(Path.Combine(basePath, destImageName)); var destImage = destBitmap .ToRgbImage() .FromRgbToLab(); var destSmallBitmap = destBitmap.CloneWithScaleTo(destBitmap.Width / 2, destBitmap.Height / 2); var destSmallImage = destSmallBitmap .ToRgbImage() .FromRgbToLab(); destBitmap.Dispose(); destSmallBitmap.Dispose(); // Init an nnf var nnf = new Nnf(destSmallImage.Width, destSmallImage.Height, srcSmallImage.Width, srcSmallImage.Height, patchSize); // Prepage setting for the PM algorithm var settings = new PatchMatchSettings { PatchSize = patchSize }; var patchMatchNnfBuilder = new PatchMatchNnfBuilder(); // Create the nnf for the small variant of the images // with a couple of iterations. patchMatchNnfBuilder.RunRandomNnfInitIteration(nnf, destSmallImage, srcSmallImage, settings); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destSmallImage, srcSmallImage, NeighboursCheckDirection.Backward, settings); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destSmallImage, srcSmallImage, NeighboursCheckDirection.Forward, settings); // The scaling of the NNF from the small images to the bigger ones. var scaledNnf = nnf.CloneAndScale2XWithUpdate(destImage, srcImage, settings); // Prepare results, save and show them scaledNnf .RestoreImage(srcImage, 3, patchSize) .FromLabToRgb() .FromRgbToBitmap() .SaveTo(@"..\..\l2r.png", ImageFormat.Png); scaledNnf .ToRgbImage() .FromRgbToBitmap() .SaveTo(@"..\..\l2n.png", ImageFormat.Png) .ShowFile(); Console.WriteLine($"NnfScaleUp processing is finished."); }
/// <summary> /// Runs the NNF build iteration for the associated areas of the dest and the source images. /// </summary> /// <param name="nnf">The NNF.</param> /// <param name="destImage">The dest image. For each patch at this image we will look for a similar one at the source image.</param> /// <param name="srcImage">The source image. Source of the patches for the dest image.</param> /// <param name="direction">The direction to look for a patches.</param> /// <param name="settings">The settings that control parameters of the algorithm.</param> /// <param name="patchDistanceCalculator">The calculator that calculates similarity of two patches. By deafult the Cie76 is used.</param> /// <param name="areasMapping">The areas mapping. By default whole area of the dest image is associated with the whole area of the source image.</param> /// <param name="destPixelsArea">Area on the dest image that actually containes pixels. By default is the area of the entire image.</param> /// <exception cref="ArgumentNullException"> /// nnf /// or /// destImage /// or /// srcImage /// or /// settings /// or /// patchDistanceCalculator /// or /// areasMapping /// or /// destPixelsArea /// </exception> public unsafe void RunBuildNnfIteration(Nnf nnf, ZsImage destImage, ZsImage srcImage, NeighboursCheckDirection direction, PatchMatchSettings settings, ImagePatchDistanceCalculator patchDistanceCalculator, Area2DMap areasMapping, Area2D destPixelsArea) { if (nnf == null) { throw new ArgumentNullException(nameof(nnf)); } if (destImage == null) { throw new ArgumentNullException(nameof(destImage)); } if (srcImage == null) { throw new ArgumentNullException(nameof(srcImage)); } if (settings == null) { throw new ArgumentNullException(nameof(settings)); } if (patchDistanceCalculator == null) { throw new ArgumentNullException(nameof(patchDistanceCalculator)); } if (areasMapping == null) { throw new ArgumentNullException(nameof(areasMapping)); } if (destPixelsArea == null) { throw new ArgumentNullException(nameof(destPixelsArea)); } sbyte offs = (sbyte)(direction == NeighboursCheckDirection.Forward ? 1 : -1); sbyte[][] offsets = { new[] { offs, (sbyte)0 }, new[] { (sbyte)0, offs } }; #region Cache settings and inputs in the local variables var nnfdata = nnf.GetNnfItems(); var nnfSrcWidth = nnf.SourceWidth; var patchSize = settings.PatchSize; var patchLength = patchSize * patchSize; var randomSearchRadius = settings.RandomSearchRadius; var alpha = settings.RandomSearchAlpha; var minSearchWindowSize = settings.MinSearchWindowSize; var destImageWidth = destImage.Width; var srcImageWidth = srcImage.Width; #endregion // Obtain the data that allow us getting a point index // regarding to the process direction(forward or backward) var addModData = GetAddModData(direction, areasMapping.DestElementsCount); var add = addModData.Item1; var mod = addModData.Item2; // Decide on how many partitions we should divade the processing // of the elements. var partsCount = areasMapping.DestElementsCount > settings.NotDividableMinAmountElements ? settings.ThreadsCount : 1; var partSize = areasMapping.DestElementsCount / partsCount; var pixelsArea = (areasMapping as IAreasMapping).DestArea; var destPointIndexes = pixelsArea.GetPointsIndexes(destImageWidth, direction == NeighboursCheckDirection.Forward); pixelsArea = pixelsArea.Intersect(destPixelsArea); var destAvailablePixelsIndexes = pixelsArea.GetPointsIndexes(destImageWidth, direction == NeighboursCheckDirection.Forward); var mappings = areasMapping.ExtractMappedAreasInfo(destImageWidth, srcImageWidth, direction == NeighboursCheckDirection.Forward); //for (int partIndex = 0; partIndex < partsCount; partIndex++) Parallel.For(0, partsCount, partIndex => { var rnd = new FastRandom(); int i; // Colne mapping to avoid conflicts in multithread //var destPointsIndexesSet = new HashSet<int>(destPointIndexes); var destAvailablePixelsIndexesSet = new HashSet <int>(destAvailablePixelsIndexes); var mappedAreasInfos = new MappedAreasInfo[mappings.Length]; for (i = 0; i < mappings.Length; i++) { mappedAreasInfos[i] = mappings[i].Clone(); } var firstPointIndex = partIndex * partSize; var lastPointIndex = firstPointIndex + partSize - 1; if (partIndex == partsCount - 1) { lastPointIndex = destPointIndexes.Length - 1; } if (lastPointIndex > destPointIndexes.Length) { lastPointIndex = destPointIndexes.Length - 1; } // Init the dest & source patch var destPatchPixelsIndexes = new int[patchSize * patchSize]; var srcPatchPixelsIndexes = new int[patchSize * patchSize]; bool isPatchFit = false; fixed(double *nnfP = nnfdata) fixed(int *destPointIndexesP = destPointIndexes) fixed(int *srcPatchPixelsIndexesP = srcPatchPixelsIndexes) fixed(int *destPatchPixelsIndexesP = destPatchPixelsIndexes) fixed(double *destImagePixelsDataP = destImage.PixelsData) fixed(double *sourceImagePixelsDataP = srcImage.PixelsData) { for (var j = firstPointIndex; j <= lastPointIndex; j++) { // Obtain a destination point from the mapping. // The dest point is the position of the dest patch. // Chocie of the destination point depends on the // direction in which we iterate the points (forward or backward). var destPointIndex = *(destPointIndexesP + add + j * mod); var destPointX = destPointIndex % destImageWidth; var destPointY = destPointIndex / destImageWidth; // We going to find the most similar source patch to the dest patch // by comparing them. // Note: we don't care about the coverage state of the dest patch // because it is at least partially covered. The situation when it is // not covered at all is impossible, since the dest point is a part of the // dest area. // We store our patch in a flat array of points for simplicity. // Populate dest patch array with corresponding dest image points indexes. Utils.PopulatePatchPixelsIndexes(destPatchPixelsIndexesP, destPointX, destPointY, patchSize, destImageWidth, destAvailablePixelsIndexesSet, out isPatchFit); // Obtain the mapped areas info with the destination area // that contains the current dest point. In the associated // source area we are allowed to look for the source patches. MappedAreasInfo mappedAreasInfo = mappedAreasInfos.FirstOrDefault(mai => mai.DestAreaPointsIndexesSet.Contains(destPointIndex)); // To improve performance, we cache the value of the NNF // at the current point in the local variables. var nnfPos = destPointIndex * 2; var bestSrcPointIndex = (int)*(nnfP + nnfPos + 0); var distanceToBestSrcPatch = *(nnfP + nnfPos + 1); // We assume that the value of the NNF at the current // point is the best so far and we have not found anything beter. var isBetterSrcPatchFound = false; // Now we check two close neighbours. First - horisontal, than vertical. #region f(x - p) + (1, 0) & f(x - p) + (0, 1) //1st iteration: f(x - p) + (1, 0) //2nd iteration: f(x - p) + (0, 1) int candidateSrcPointY = 0; int candidateSrcPointX = 0; double distance; for (var offsetIndex = 0; offsetIndex < offsets.Length; offsetIndex++) { // Obtain the position of the neighbour. // During the forward search // on the first iteration it is the neighbour at the left side // on the second iteration it is the neighbour at the top // During the backward search // on the first iteration it is the neighbour at the right side // on the second iteration it is the neighbour at the bottom var offset = offsets[offsetIndex]; var destPointNeighbourX = destPointX - offset[0]; var destPointNeighbourY = destPointY - offset[1]; var destPointNeighbourIndex = destPointNeighbourY * destImageWidth + destPointNeighbourX; // Make sure that the neighbour point belongs to the mapping if (!mappedAreasInfo.DestAreaPointsIndexesSet.Contains(destPointNeighbourIndex)) { continue; } // There is a high chance that the neighbour's source patch neighbour // is similar to the current dest patch. Let's check it. // Obtain the position of the neighbour's source patch neighbour. var neighbourNnfPos = 2 * destPointNeighbourIndex; var neighbourSrcPointIndex = (int)*(nnfP + neighbourNnfPos + 0); candidateSrcPointY = neighbourSrcPointIndex / nnfSrcWidth + offset[1]; candidateSrcPointX = neighbourSrcPointIndex % nnfSrcWidth + offset[0]; var candidateSrcPointIndex = candidateSrcPointY * srcImageWidth + candidateSrcPointX; // Make sure that the source patch resides inside the // associated source area of the dest area. if (!mappedAreasInfo.SrcAreaPointsIndexesSet.Contains(candidateSrcPointIndex)) { continue; } // Populate source patch array with corresponding source image points indexes Utils.PopulatePatchPixelsIndexes(srcPatchPixelsIndexesP, candidateSrcPointX, candidateSrcPointY, patchSize, srcImageWidth, mappedAreasInfo.SrcAreaPointsIndexesSet, out isPatchFit); // Calculate distance between the patches distance = patchDistanceCalculator.Calculate(destPatchPixelsIndexesP, srcPatchPixelsIndexesP, distanceToBestSrcPatch, destImagePixelsDataP, sourceImagePixelsDataP, destImage, srcImage, patchLength); if (isPatchFit && distance < distanceToBestSrcPatch) { isBetterSrcPatchFound = true; bestSrcPointIndex = candidateSrcPointIndex; distanceToBestSrcPatch = distance; } } #endregion #region random search //v0 = f(x) // For the random search we need to find out // the size of the window where to search var srcAreaBound = mappedAreasInfo.SrcBound; var srcAreaWidth = srcAreaBound.Width; var srcAreaHeight = srcAreaBound.Height; var searchWindowSize = srcAreaHeight < srcAreaWidth ? srcAreaHeight : srcAreaWidth; if (searchWindowSize > minSearchWindowSize) { searchWindowSize = minSearchWindowSize; } // Init the search radius; var searchRadius = randomSearchRadius; var bestSrcPointX = bestSrcPointIndex % srcImageWidth; var bestSrcPointY = bestSrcPointIndex / srcImageWidth; for (i = 0; searchRadius >= 1; i++) { //TODO: change and measure inpact of changing call to Pow searchRadius = searchWindowSize * System.Math.Pow(alpha, i); isPatchFit = false; var attempts = 0; while (!isPatchFit && attempts < 5) { attempts++; candidateSrcPointX = (int)(bestSrcPointX + (rnd.NextDouble() * 2 - 1.0) * searchRadius); candidateSrcPointY = (int)(bestSrcPointY + (rnd.NextDouble() * 2 - 1.0) * searchRadius); // Populate source patch array with corresponding source image points indexes Utils.PopulatePatchPixelsIndexes(srcPatchPixelsIndexesP, candidateSrcPointX, candidateSrcPointY, patchSize, srcImageWidth, mappedAreasInfo.SrcAreaPointsIndexesSet, out isPatchFit); } // Calculate distance between the patches distance = patchDistanceCalculator.Calculate(destPatchPixelsIndexesP, srcPatchPixelsIndexesP, distanceToBestSrcPatch, destImagePixelsDataP, sourceImagePixelsDataP, destImage, srcImage, patchLength); if (isPatchFit && distance < distanceToBestSrcPatch) { distanceToBestSrcPatch = distance; bestSrcPointIndex = candidateSrcPointY * srcImageWidth + candidateSrcPointX; isBetterSrcPatchFound = true; } } #endregion if (isBetterSrcPatchFound) { *(nnfP + nnfPos + 0) = bestSrcPointIndex; *(nnfP + nnfPos + 1) = distanceToBestSrcPatch; } } } }); }
private ZsImage InternalInpaint(Pyramid pyramid, InpaintSettings settings) { #region cache settings var patchSize = settings.PatchSize; var calculator = settings.PatchDistanceCalculator; var nnfSettings = settings.PatchMatch; var changedPixelsPercentTreshold = settings.ChangedPixelsPercentTreshold; var maxInpaintIterationsAmount = settings.MaxInpaintIterations; var kStep = settings.MeanShift.KDecreaseStep; var minK = settings.MeanShift.MinK; #endregion ZsImage image = null; // go thru all the pyramid levels starting from the top one Nnf nnf = null; for (byte levelIndex = 0; levelIndex < pyramid.LevelsAmount; levelIndex++) { image = pyramid.GetImage(levelIndex); var mapping = pyramid.GetMapping(levelIndex); var inpaintArea = pyramid.GetInpaintArea(levelIndex); var imageArea = Area2D.Create(0, 0, image.Width, image.Height); // if there is a NNF built on the prev level // scale it up nnf = nnf == null ? new Nnf(image.Width, image.Height, image.Width, image.Height, patchSize) : nnf.CloneAndScale2XWithUpdate(image, image, nnfSettings, mapping, calculator); // start inpaint iterations var k = settings.MeanShift.K; for (var inpaintIterationIndex = 0; inpaintIterationIndex < maxInpaintIterationsAmount; inpaintIterationIndex++) { // Obtain pixels area. // Pixels area defines which pixels are allowed to be used // for the patches distance calculation. We must avoid pixels // that we want to inpaint. That is why before the area is not // inpainted - we should exclude this area. var pixelsArea = imageArea; if (levelIndex == 0 && inpaintIterationIndex == 0 && settings.IgnoreInpaintedPixelsOnFirstIteration) { pixelsArea = imageArea.Substract(inpaintArea); } // skip building NNF for the first iteration in the level // unless it is top level (for the top one we haven't built NNF yet) if (levelIndex == 0 || inpaintIterationIndex > 0) { // in order to find best matches for the inpainted area, // we build NNF for this image as a dest and a source // but excluding the inpainted area from the source area // (our mapping already takes care of it) _nnfBuilder.RunRandomNnfInitIteration(nnf, image, image, nnfSettings, calculator, mapping, pixelsArea); _nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Forward, nnfSettings, calculator, mapping, pixelsArea); _nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Backward, nnfSettings, calculator, mapping, pixelsArea); _nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Forward, nnfSettings, calculator, mapping, pixelsArea); _nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Backward, nnfSettings, calculator, mapping, pixelsArea); _nnfBuilder.RunBuildNnfIteration(nnf, image, image, NeighboursCheckDirection.Forward, nnfSettings, calculator, mapping, pixelsArea); } var nnfNormalized = nnf.Clone(); nnfNormalized.Normalize(); // after we have the NNF - we calculate the values of the pixels in the inpainted area var inpaintResult = Inpaint(image, inpaintArea, nnfNormalized, k, settings); k = k > minK ? k - kStep : k; if (IterationFinished != null) { var eventArgs = new InpaintIterationFinishedEventArgs { InpaintedLabImage = image.Clone(), InpaintResult = inpaintResult, LevelIndex = levelIndex, InpaintIteration = inpaintIterationIndex }; IterationFinished(this, eventArgs); } // if the change is smaller then a treshold, we quit if (inpaintResult.ChangedPixelsPercent < changedPixelsPercentTreshold) { break; } //if (levelIndex == pyramid.LevelsAmount - 1) break; } } return(image); }
static void Main(string[] args) { const string basePath = "..\\..\\..\\..\\images"; var destImageName = "pm1.png"; var srcImageName = "pm2.png"; var destArea1ImageName = "pm1_target1.png"; var destArea2ImageName = "pm1_target2.png"; // this is our input data. var destImage = GetLabImage(basePath, destImageName); var srcImage = GetLabImage(basePath, srcImageName); var destArea1 = GetArea2D(basePath, destArea1ImageName); var destArea2 = GetArea2D(basePath, destArea2ImageName); var srcArea = Area2D.Create(0, 0, srcImage.Width, srcImage.Height); var map1 = new Area2DMapBuilder() .InitNewMap(destArea1, srcArea) .Build(); var map2 = new Area2DMapBuilder() .InitNewMap(destArea2, srcArea) .Build(); var settings = new PatchMatchSettings { PatchSize = 5 }; var patchMatchNnfBuilder = new PatchMatchNnfBuilder(); var calculator = ImagePatchDistance.Cie76; // Create nnf for the images // with a couple of iterations. var nnf1 = new Nnf(destImage.Width, destImage.Height, srcImage.Width, srcImage.Height, settings.PatchSize); patchMatchNnfBuilder.RunRandomNnfInitIteration(nnf1, destImage, srcImage, settings, calculator, map1); patchMatchNnfBuilder.RunBuildNnfIteration(nnf1, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map1); patchMatchNnfBuilder.RunBuildNnfIteration(nnf1, destImage, srcImage, NeighboursCheckDirection.Backward, settings, calculator, map1); patchMatchNnfBuilder.RunBuildNnfIteration(nnf1, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map1); patchMatchNnfBuilder.RunBuildNnfIteration(nnf1, destImage, srcImage, NeighboursCheckDirection.Backward, settings, calculator, map1); patchMatchNnfBuilder.RunBuildNnfIteration(nnf1, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map1); // Create the nnf for the images // with a couple of iterations. var nnf2 = new Nnf(destImage.Width, destImage.Height, srcImage.Width, srcImage.Height, settings.PatchSize); patchMatchNnfBuilder.RunRandomNnfInitIteration(nnf2, destImage, srcImage, settings, calculator, map2); patchMatchNnfBuilder.RunBuildNnfIteration(nnf2, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map2); patchMatchNnfBuilder.RunBuildNnfIteration(nnf2, destImage, srcImage, NeighboursCheckDirection.Backward, settings, calculator, map2); patchMatchNnfBuilder.RunBuildNnfIteration(nnf2, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map2); patchMatchNnfBuilder.RunBuildNnfIteration(nnf2, destImage, srcImage, NeighboursCheckDirection.Backward, settings, calculator, map2); patchMatchNnfBuilder.RunBuildNnfIteration(nnf2, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map2); // Show the built NNFs and restored images nnf2 .RestoreImage(srcImage, 3, settings.PatchSize) .FromLabToRgb() .FromRgbToBitmap() .SaveTo(@"..\..\..\restored1.png", ImageFormat.Png); nnf1 .RestoreImage(srcImage, 3, settings.PatchSize) .FromLabToRgb() .FromRgbToBitmap() .SaveTo(@"..\..\..\restored2.png", ImageFormat.Png); nnf1 .ToRgbImage() .FromRgbToBitmap() .SaveTo(@"..\..\..\nnf1.png", ImageFormat.Png); nnf2 .ToRgbImage() .FromRgbToBitmap() .SaveTo(@"..\..\..\nnf2.png", ImageFormat.Png); // Let's now merge the built NNFs and try to restore an image nnf2.Merge(nnf1, map2, map1); nnf2 .RestoreImage(srcImage, 3, settings.PatchSize) .FromLabToRgb() .FromRgbToBitmap() .SaveTo(@"..\..\..\restored_whole.png", ImageFormat.Png); nnf2 .ToRgbImage() .FromRgbToBitmap() .SaveTo(@"..\..\..\merged_nnf.png", ImageFormat.Png) .ShowFile(); Console.WriteLine($"PatchMatchPipeline processing is finished."); }
static void Main(string[] args) { var sw = new Stopwatch(); sw.Start(); const string basePath = "..\\..\\..\\..\\images"; var destImageName = "pm1.png"; var srcImageName = "pm2.png"; var destTargetImageName = "dd1.png"; // this is our input data. var destImage = GetLabImage(basePath, destImageName); var srcImage = GetLabImage(basePath, srcImageName); var destTargetArea1 = GetArea2D(basePath, destTargetImageName); var srcArea1 = GetArea2D(basePath, "sd1.png"); var destArea2 = GetArea2D(basePath, "dd2.png"); var srcArea2 = GetArea2D(basePath, "sd2.png"); var destArea = Area2D.Create(0, 0, destImage.Width, destImage.Height); var srcArea = Area2D.Create(0, 0, srcImage.Width, srcImage.Height); var map = new Area2DMapBuilder() .InitNewMap(destArea, srcArea) .AddAssociatedAreas(destTargetArea1, srcArea1) .AddAssociatedAreas(destArea2, srcArea2) .Build(); const byte patchSize = 5; var nnf = new Nnf(destImage.Width, destImage.Height, srcImage.Width, srcImage.Height, patchSize); // Prepage setting for the PM algorithm var settings = new PatchMatchSettings { PatchSize = patchSize }; var calculator = ImagePatchDistance.Cie76; var patchMatchNnfBuilder = new PatchMatchNnfBuilder(); // Create the nnf for the small variant of the images // with a couple of iterations. patchMatchNnfBuilder.RunRandomNnfInitIteration(nnf, destImage, srcImage, settings, calculator, map); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Backward, settings, calculator, map); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Backward, settings, calculator, map); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map); // Restore dest image from the NNF and source image. nnf .RestoreImage(srcImage, 3, patchSize) .FromLabToRgb() .FromRgbToBitmap() .SaveTo(@"..\..\..\restored.png", ImageFormat.Png); // Convert the NNF to an image, save and show it nnf .ToRgbImage() .FromRgbToBitmap() .SaveTo(@"..\..\..\nnf.png", ImageFormat.Png) .ShowFile(); sw.Stop(); Console.WriteLine($"Elapsed time: {sw.Elapsed}"); Console.WriteLine($"PatchMatchPipeline processing is finished."); }
/// <summary> /// Merges provided NNF into. /// </summary> /// <param name="destNnf">The dest NNF.</param> /// <param name="srcNnf">The source NNF.</param> /// <param name="destNnfMap">The dest NNF areas mapping.</param> /// <param name="srcNnfMap">The source NNF areas mapping.</param> /// <param name="options">The parallel processing options.</param> /// <exception cref="ArgumentNullException"> /// destNnf /// or /// srcNnf /// or /// destNnfMap /// or /// srcNnfMap /// </exception> /// <exception cref="ArgumentException">NNFs should be built for the same source and dest images</exception> public static unsafe void Merge(this Nnf destNnf, Nnf srcNnf, Area2DMap destNnfMap, Area2DMap srcNnfMap, ParallelOptions options = null) { if (destNnf == null) { throw new ArgumentNullException(nameof(destNnf)); } if (srcNnf == null) { throw new ArgumentNullException(nameof(srcNnf)); } if (destNnfMap == null) { throw new ArgumentNullException(nameof(destNnfMap)); } if (srcNnfMap == null) { throw new ArgumentNullException(nameof(srcNnfMap)); } // We assume that all the inputs contain NNFs // for the same dest image. // The source images can not be different as well. // When different source images are desired those // images should be merged to one image and mappings // should be adjusted to map to a particular region // on the source image. // Differen source images problem. In that case we // have a problem with different mappings when we // merge NNfs. Mappings can not be merged as they // would have totally different source areas. // make sure that both NNFs are built for the same dest and source images if (destNnf.DstWidth != srcNnf.DstWidth || destNnf.DstHeight != srcNnf.DstHeight || destNnf.SourceWidth != srcNnf.SourceWidth || destNnf.SourceHeight != srcNnf.SourceHeight) { throw new ArgumentException("NNFs should be built for the same source and dest images"); } if (options == null) { options = new ParallelOptions(); } var destImageWidth = destNnf.DstWidth; var srcImageWidth = destNnf.SourceWidth; // Nnf that needs to be merged to our dest nnf. var srcNnfData = srcNnf.GetNnfItems(); var destNnfData = destNnf.GetNnfItems(); var destNnfPointsIndexes = (destNnfMap as IAreasMapping).DestArea.GetPointsIndexes(destImageWidth); var srcNnfPointsIndexes = (srcNnfMap as IAreasMapping).DestArea.GetPointsIndexes(destImageWidth); var mappings = srcNnfMap.ExtractMappedAreasInfo(destImageWidth, srcImageWidth); // Decide on how many partitions we should divade the processing // of the elements. int partsCount = srcNnfMap.DestElementsCount > options.NotDividableMinAmountElements ? options.ThreadsCount : 1; var partSize = (int)(srcNnfMap.DestElementsCount / partsCount); Parallel.For(0, partsCount, partIndex => { // Colne mapping to avoid conflicts in multithread var destNnfPointsIndexesSet = new HashSet <int>(destNnfPointsIndexes); // Clone mappings to avoid problems in multithread var mappedAreasInfos = new MappedAreasInfo[mappings.Length]; for (int j = 0; j < mappings.Length; j++) { mappedAreasInfos[j] = mappings[j].Clone(); } var firstPointIndex = partIndex * partSize; var lastPointIndex = firstPointIndex + partSize - 1; if (partIndex == partsCount - 1) { lastPointIndex = srcNnfMap.DestElementsCount - 1; } if (lastPointIndex > srcNnfMap.DestElementsCount) { lastPointIndex = srcNnfMap.DestElementsCount - 1; } fixed(double *destNnfP = destNnfData) fixed(double *srcNnfP = srcNnfData) fixed(int *srcNnfPointIndexesP = srcNnfPointsIndexes) { for (var srcNnfMapDestPointIndex = firstPointIndex; srcNnfMapDestPointIndex <= lastPointIndex; srcNnfMapDestPointIndex++) { var destPointIndex = *(srcNnfPointIndexesP + srcNnfMapDestPointIndex); if (destNnfPointsIndexesSet.Contains(destPointIndex)) { // The value of the NNF in the dest point can // present in the resulting destNnf as well. // In that case we need to merge NNFs at the point // by taking the best value. // compare and set the best var srcVal = *(srcNnfP + destPointIndex * 2 + 1); var destVal = *(destNnfP + destPointIndex * 2 + 1); if (srcVal < destVal) { *(destNnfP + destPointIndex * 2 + 0) = *(srcNnfP + destPointIndex * 2 + 0); *(destNnfP + destPointIndex * 2 + 1) = srcVal; } } else { // When the destNnf doesn't contain the value // for that point we simply copy it *(destNnfP + destPointIndex * 2 + 0) = *(srcNnfP + destPointIndex * 2 + 0); *(destNnfP + destPointIndex * 2 + 1) = *(srcNnfP + destPointIndex * 2 + 1); } } } }); }
/// <summary> /// Runs the random NNF initialization iteration for the associated areas of the dest and the source images. /// </summary> /// <param name="nnf">The NNF.</param> /// <param name="destImage">The dest image. For each patch at this image we will look for a similar one at the source image.</param> /// <param name="srcImage">The source image. Source of the patches for the dest image.</param> /// <param name="settings">The settings that control parameters of the algorithm.</param> /// <param name="patchDistanceCalculator">The calculator that calculates similarity of two patches. By deafult the Cie76 is used.</param> /// <param name="areasMapping">The areas mapping. By default whole area of the dest image is associated with the whole area of the source image.</param> /// <param name="destPixelsArea">Area on the dest image that actually containes pixels. By default is the area of the entire image.</param> /// <exception cref="ArgumentNullException"> /// nnf /// or /// destImage /// or /// srcImage /// or /// settings /// or /// patchDistanceCalculator /// or /// areasMapping /// or /// destPixelsArea /// </exception> public unsafe void RunRandomNnfInitIteration(Nnf nnf, ZsImage destImage, ZsImage srcImage, PatchMatchSettings settings, ImagePatchDistanceCalculator patchDistanceCalculator, Area2DMap areasMapping, Area2D destPixelsArea) { if (nnf == null) { throw new ArgumentNullException(nameof(nnf)); } if (destImage == null) { throw new ArgumentNullException(nameof(destImage)); } if (srcImage == null) { throw new ArgumentNullException(nameof(srcImage)); } if (settings == null) { throw new ArgumentNullException(nameof(settings)); } if (patchDistanceCalculator == null) { throw new ArgumentNullException(nameof(patchDistanceCalculator)); } if (areasMapping == null) { throw new ArgumentNullException(nameof(areasMapping)); } if (destPixelsArea == null) { throw new ArgumentNullException(nameof(destPixelsArea)); } const byte MaxAttempts = 32; var nnfdata = nnf.GetNnfItems(); var patchSize = settings.PatchSize; var patchPointsAmount = patchSize * patchSize; var destImageWidth = destImage.Width; var srcImageWidth = srcImage.Width; // Decide on how many partitions we should divade the processing // of the elements. var partsCount = areasMapping.DestElementsCount > settings.NotDividableMinAmountElements ? settings.ThreadsCount : 1; var partSize = (int)(areasMapping.DestElementsCount / partsCount); var pixelsArea = (areasMapping as IAreasMapping).DestArea; var destPointsIndexes = pixelsArea.GetPointsIndexes(destImageWidth); pixelsArea = pixelsArea.Intersect(destPixelsArea); var destAvailablePixelsIndexes = pixelsArea.GetPointsIndexes(destImageWidth); var mappings = areasMapping.ExtractMappedAreasInfo(destImageWidth, srcImageWidth); //for (int partIndex = 0; partIndex < partsCount; partIndex++) Parallel.For(0, partsCount, partIndex => { // Colne mapping to avoid conflicts in multithread //var destPointsIndexesSet = new HashSet<int>(destPointsIndexes); var destAvailablePixelsIndexesSet = new HashSet <int>(destAvailablePixelsIndexes); var mappedAreasInfos = new MappedAreasInfo[mappings.Length]; for (int i = 0; i < mappings.Length; i++) { mappedAreasInfos[i] = mappings[i].Clone(); } #region Variables definition int destPointIndex; int destPointX; int destPointY; int srcPointIndex = 0; int srcPointX; int srcPointY; double distance; int nnfPos; byte attemptsCount = 0; #endregion var rnd = new FastRandom(); var firstPointIndex = partIndex * partSize; var lastPointIndex = firstPointIndex + partSize - 1; if (partIndex == partsCount - 1) { lastPointIndex = areasMapping.DestElementsCount - 1; } if (lastPointIndex > areasMapping.DestElementsCount) { lastPointIndex = areasMapping.DestElementsCount - 1; } // Init the dest & source patch var destPatchPointIndexes = new int[patchPointsAmount]; var srcPatchPointIndexes = new int[patchPointsAmount]; bool isPatchFit = false; fixed(double *nnfdataP = nnfdata) fixed(int *dstPointIndexesP = destPointsIndexes) fixed(int *srcPatchPointIndexesP = srcPatchPointIndexes) fixed(int *destPatchPointIndexesP = destPatchPointIndexes) fixed(double *destPixelsDataP = destImage.PixelsData) fixed(double *sourcePixelsDataP = srcImage.PixelsData) { for (var pointIndex = firstPointIndex; pointIndex <= lastPointIndex; pointIndex++) { destPointIndex = *(dstPointIndexesP + pointIndex); destPointX = destPointIndex % destImageWidth; destPointY = destPointIndex / destImageWidth; Utils.PopulatePatchPixelsIndexes(destPatchPointIndexesP, destPointX, destPointY, patchSize, destImageWidth, destAvailablePixelsIndexesSet, out isPatchFit); nnfPos = destPointIndex * 2; // Obtain the source area associated with the destination area // to which the current dest point belongs to. // In that area we are allowed to look for the source patches. MappedAreasInfo mappedAreasInfo = mappedAreasInfos.FirstOrDefault(mai => mai.DestAreaPointsIndexesSet.Contains(destPointIndex)); isPatchFit = false; attemptsCount = 0; var srcPointsAmount = mappedAreasInfo.SrcAreaPointsIndexes.Length; while (!isPatchFit && attemptsCount < MaxAttempts && attemptsCount < srcPointsAmount) { srcPointIndex = mappedAreasInfo.SrcAreaPointsIndexes[rnd.Next(srcPointsAmount)]; srcPointX = srcPointIndex % srcImageWidth; srcPointY = srcPointIndex / srcImageWidth; Utils.PopulatePatchPixelsIndexes(srcPatchPointIndexesP, srcPointX, srcPointY, patchSize, srcImageWidth, mappedAreasInfo.SrcAreaPointsIndexesSet, out isPatchFit); attemptsCount++; } distance = patchDistanceCalculator.Calculate(destPatchPointIndexesP, srcPatchPointIndexesP, double.MaxValue, destPixelsDataP, sourcePixelsDataP, destImage, srcImage, patchPointsAmount); *(nnfdataP + nnfPos + 0) = srcPointIndex; *(nnfdataP + nnfPos + 1) = distance; } } }); }
/// <summary> /// Runs the random NNF initialization iteration using the whole areas of the dest and the source images. Uses CIE76 to calculate patch similarity. /// </summary> /// <param name="nnf">The NNF.</param> /// <param name="destImage">The dest image. For each patch at this image we will look for a similar one at the source image.</param> /// <param name="srcImage">The source image. Source of the patches for the dest image.</param> /// <param name="settings">The settings that control parameters of the algorithm.</param> public void RunRandomNnfInitIteration(Nnf nnf, ZsImage destImage, ZsImage srcImage, PatchMatchSettings settings) { var patchDistanceCalculator = ImagePatchDistance.Cie76; RunRandomNnfInitIteration(nnf, destImage, srcImage, settings, patchDistanceCalculator); }
private InpaintingResult Inpaint(ZsImage image, Area2D removeArea, Nnf nnf, double k, IInpaintSettings settings) { // Since the nnf was normalized, the sigma now is normalized as well and it // has not any more some specific value. //const double naturalLogBase = System.Math.E; //const double minusSigmaCube2 = -0.91125;//-2 * sigma * sigma; sigma = 0.675 = 75percentil //double NaturalLogBase2 = System.Math.Pow(System.Math.E, 1.0 / minusSigmaCube2); const double naturalLogBasePowMinusSigmaCube2 = 0.33373978049163078; const double maxSquareDistanceInLab = 32668.1151; // Gamma is used for calculation of alpha in markup. The confidence. const double gamma = 1.3; // The value of confidence in non marked areas. const double confidentValue = 1.50; var patchSize = settings.PatchSize; var colorResolver = settings.ColorResolver; var pixelChangeTreshold = settings.PixelChangeTreshold; // Get points' indexes that that needs to be inpainted. var pointIndexes = new int[removeArea.ElementsCount]; removeArea.FillMappedPointsIndexes(pointIndexes, image.Width); var patchOffset = (patchSize - 1) / 2; var imageWidth = image.Width; var nnfWidth = nnf.DstWidth; var nnfHeight = nnf.DstHeight; var srcWidth = nnf.SourceWidth; var srcHeight = nnf.SourceHeight; var nnfdata = nnf.GetNnfItems(); var cmpts = image.NumberOfComponents; // Weighted color is color's components values + weight of the color. var weightedColors = new double[patchSize * patchSize * (cmpts + 1)]; var confidence = removeArea.CalculatePointsConfidence(confidentValue, gamma); var resolvedColor = new double[cmpts]; var totalDifference = 0.0; var changedPixelsAmount = 0; var changedPixelsDifference = 0.0; for (int ii = 0; ii < pointIndexes.Length; ii++) { var pointIndex = pointIndexes[ii]; int pty = pointIndex / imageWidth; int ptx = pointIndex % imageWidth; int colorsCount = 0; //go thru the patch for (int y = pty - patchOffset, j = 0; j < patchSize; j++, y++) { for (int x = ptx - patchOffset, i = 0; i < patchSize; i++, x++) { if (0 <= x && x < nnfWidth && 0 <= y && y < nnfHeight) { int destPatchPointIndex = y * nnfWidth + x; int srcPatchPointIndex = (int)nnfdata[destPatchPointIndex * 2]; int srcPatchPointX = srcPatchPointIndex % nnfWidth; int srcPatchPointY = srcPatchPointIndex / nnfWidth; int srcPixelX = srcPatchPointX - i + patchOffset; int srcPixelY = srcPatchPointY - j + patchOffset; if (0 <= srcPixelX && srcPixelX < srcWidth && 0 <= srcPixelY && srcPixelY < srcHeight) { int srcPixelIndex = srcPixelY * imageWidth + srcPixelX; for (int ci = 0; ci < cmpts; ci++) { weightedColors[colorsCount * (cmpts + 1) + ci] = image.PixelsData[srcPixelIndex * cmpts + ci]; } weightedColors[colorsCount * (cmpts + 1) + cmpts] = System.Math.Pow(naturalLogBasePowMinusSigmaCube2, nnfdata[destPatchPointIndex * 2 + 1]) * confidence[ii]; colorsCount++; } } } } // calculate color colorResolver.Resolve(weightedColors, 0, colorsCount, cmpts, k, resolvedColor, 0); // how different is resolvedColor from the old color? var labL = resolvedColor[0] - image.PixelsData[pointIndex * cmpts + 0]; var labA = resolvedColor[1] - image.PixelsData[pointIndex * cmpts + 1]; var labB = resolvedColor[2] - image.PixelsData[pointIndex * cmpts + 2]; var pixelsDiff = (labL * labL + labA * labA + labB * labB) / maxSquareDistanceInLab; totalDifference += pixelsDiff; if (pixelsDiff > pixelChangeTreshold) { changedPixelsAmount++; changedPixelsDifference += pixelsDiff; } for (var i = 0; i < cmpts; i++) { image.PixelsData[pointIndex * cmpts + i] = resolvedColor[i]; } } totalDifference /= pointIndexes.Length; _inpaintingResult.TotalDifference = totalDifference / pointIndexes.Length; _inpaintingResult.PixelsChangedAmount = changedPixelsAmount; _inpaintingResult.PixelsToInpaintAmount = pointIndexes.Length; _inpaintingResult.ChangedPixelsDifference = changedPixelsDifference; return(_inpaintingResult); }
static void Main(string[] args) { const string basePath = "..\\..\\..\\..\\images"; const int patchSize = 5; var srcImageName = "t009.jpg"; // Prepare images var srcImage = GetLabImage(basePath, srcImageName); var destImage = GetLabImage(basePath, srcImageName); var ignoreArea = GetArea2D(basePath, "m009.png"); var destArea = ignoreArea.Dilation(patchSize * 2 + 1); // Init an nnf var nnf = new Nnf(destImage.Width, destImage.Height, srcImage.Width, srcImage.Height, patchSize); // Create a mapping of the areas on the dest and source areas. var imageArea = Area2D.Create(0, 0, srcImage.Width, srcImage.Height); var map = new Area2DMapBuilder() .InitNewMap(imageArea, imageArea) .SetIgnoredSourcedArea(ignoreArea) .Build(); // Prepage setting for the PM algorithm var settings = new PatchMatchSettings { PatchSize = patchSize }; var calculator = ImagePatchDistance.Cie76; var patchMatchNnfBuilder = new PatchMatchNnfBuilder(); // Create the nnf for the image(while ignoring some area) // with a couple of iterations. patchMatchNnfBuilder.RunRandomNnfInitIteration(nnf, destImage, srcImage, settings, calculator, map); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Backward, settings, calculator, map); // Create a mapping for the area that is a bit bigger // then ignored area. map = new Area2DMapBuilder() .InitNewMap(imageArea, imageArea) .ReduceDestArea(destArea) .SetIgnoredSourcedArea(ignoreArea) .Build(); patchMatchNnfBuilder.RunRandomNnfInitIteration(nnf, destImage, srcImage, settings, calculator, map); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Forward, settings, calculator, map); patchMatchNnfBuilder.RunBuildNnfIteration(nnf, destImage, srcImage, NeighboursCheckDirection.Backward, settings, calculator, map); string fileName1 = @"..\..\..\nnf1_pure.png"; nnf .ToRgbImage() .FromRgbToBitmap() .SaveTo(fileName1, ImageFormat.Png); // Normalize the NNF in the ignored area. nnf.Normalize(ignoreArea); // Prepare results, save and show them string fileName2 = @"..\..\..\nnf2_normalized.png"; nnf .ToRgbImage() .FromRgbToBitmap() .SaveTo(fileName2, ImageFormat.Png) .ShowFile(); Console.WriteLine($"Nnf normalization is finished."); }