/// <summary>
/// Normalizes the NNF in the specified dest area.
/// </summary>
/// <param name="nnf">The NNF.</param>
/// <param name="destArea">The dest area.</param>
/// <exception cref="ArgumentNullException">
/// nnf
/// or
/// destArea
/// </exception>
public static void Normalize(this Nnf nnf, Area2D destArea)
{
if (nnf == null)
{
throw new ArgumentNullException(nameof(nnf));
}
if (destArea == null)
{
throw new ArgumentNullException(nameof(destArea));
}
var nnfdata = nnf.GetNnfItems();
checked
{
var distances = new double[destArea.ElementsCount];
var pointIndexes = new int[destArea.ElementsCount];
destArea.FillMappedPointsIndexes(pointIndexes, nnf.DstWidth);
var dinstancesSum = 0.0;
for (int destPointIndex = 0; destPointIndex < distances.Length; destPointIndex++)
{
var nnfPos = pointIndexes[destPointIndex] * 2;
double distance = nnfdata[nnfPos + 1];
distances[destPointIndex] = distance;
dinstancesSum += distance;
}
double mean = dinstancesSum / distances.Length;
var squareDistances = new double[distances.Length];
double squreDistancesSum = 0.0;
for (int i = 0; i < distances.Length; i++)
{
var distToMean = distances[i] - mean;
double distToMeanCube = distToMean * distToMean;
squareDistances[i] = distToMeanCube;
squreDistancesSum += distToMeanCube;
}
double sigma = System.Math.Sqrt(squreDistancesSum / (squareDistances.Length - 1));
for (int destPointIndex = 0; destPointIndex < distances.Length; destPointIndex++)
{
var nnfPos = pointIndexes[destPointIndex] * 2;
var dist = distances[destPointIndex];
dist = (dist - mean) / sigma;
if (dist < 0)
{
dist = -dist;
}
nnfdata[nnfPos + 1] = dist;
}
}
}
/// <summary>
/// Merges provided NNF into.
/// </summary>
/// <param name="destNnf">The dest NNF.</param>
/// <param name="srcNnf">The source NNF.</param>
/// <param name="destNnfMap">The dest NNF areas mapping.</param>
/// <param name="srcNnfMap">The source NNF areas mapping.</param>
/// <param name="options">The parallel processing options.</param>
/// <exception cref="ArgumentNullException">
/// destNnf
/// or
/// srcNnf
/// or
/// destNnfMap
/// or
/// srcNnfMap
/// </exception>
/// <exception cref="ArgumentException">NNFs should be built for the same source and dest images</exception>
public static unsafe void Merge(this Nnf destNnf, Nnf srcNnf, Area2DMap destNnfMap, Area2DMap srcNnfMap, ParallelOptions options = null)
{
if (destNnf == null)
{
throw new ArgumentNullException(nameof(destNnf));
}
if (srcNnf == null)
{
throw new ArgumentNullException(nameof(srcNnf));
}
if (destNnfMap == null)
{
throw new ArgumentNullException(nameof(destNnfMap));
}
if (srcNnfMap == null)
{
throw new ArgumentNullException(nameof(srcNnfMap));
}
// We assume that all the inputs contain NNFs
// for the same dest image.
// The source images can not be different as well.
// When different source images are desired those
// images should be merged to one image and mappings
// should be adjusted to map to a particular region
// on the source image.
// Differen source images problem. In that case we
// have a problem with different mappings when we
// merge NNfs. Mappings can not be merged as they
// would have totally different source areas.
// make sure that both NNFs are built for the same dest and source images
if (destNnf.DstWidth != srcNnf.DstWidth || destNnf.DstHeight != srcNnf.DstHeight ||
destNnf.SourceWidth != srcNnf.SourceWidth || destNnf.SourceHeight != srcNnf.SourceHeight)
{
throw new ArgumentException("NNFs should be built for the same source and dest images");
}
if (options == null)
{
options = new ParallelOptions();
}
var destImageWidth = destNnf.DstWidth;
var srcImageWidth = destNnf.SourceWidth;
// Nnf that needs to be merged to our dest nnf.
var srcNnfData = srcNnf.GetNnfItems();
var destNnfData = destNnf.GetNnfItems();
var destNnfPointsIndexes = (destNnfMap as IAreasMapping).DestArea.GetPointsIndexes(destImageWidth);
var srcNnfPointsIndexes = (srcNnfMap as IAreasMapping).DestArea.GetPointsIndexes(destImageWidth);
var mappings = srcNnfMap.ExtractMappedAreasInfo(destImageWidth, srcImageWidth);
// Decide on how many partitions we should divade the processing
// of the elements.
int partsCount = srcNnfMap.DestElementsCount > options.NotDividableMinAmountElements
? options.ThreadsCount
: 1;
var partSize = (int)(srcNnfMap.DestElementsCount / partsCount);
Parallel.For(0, partsCount, partIndex =>
{
// Colne mapping to avoid conflicts in multithread
var destNnfPointsIndexesSet = new HashSet <int>(destNnfPointsIndexes);
// Clone mappings to avoid problems in multithread
var mappedAreasInfos = new MappedAreasInfo[mappings.Length];
for (int j = 0; j < mappings.Length; j++)
{
mappedAreasInfos[j] = mappings[j].Clone();
}
var firstPointIndex = partIndex * partSize;
var lastPointIndex = firstPointIndex + partSize - 1;
if (partIndex == partsCount - 1)
{
lastPointIndex = srcNnfMap.DestElementsCount - 1;
}
if (lastPointIndex > srcNnfMap.DestElementsCount)
{
lastPointIndex = srcNnfMap.DestElementsCount - 1;
}
fixed(double *destNnfP = destNnfData)
fixed(double *srcNnfP = srcNnfData)
fixed(int *srcNnfPointIndexesP = srcNnfPointsIndexes)
{
for (var srcNnfMapDestPointIndex = firstPointIndex; srcNnfMapDestPointIndex <= lastPointIndex; srcNnfMapDestPointIndex++)
{
var destPointIndex = *(srcNnfPointIndexesP + srcNnfMapDestPointIndex);
if (destNnfPointsIndexesSet.Contains(destPointIndex))
{
// The value of the NNF in the dest point can
// present in the resulting destNnf as well.
// In that case we need to merge NNFs at the point
// by taking the best value.
// compare and set the best
var srcVal = *(srcNnfP + destPointIndex * 2 + 1);
var destVal = *(destNnfP + destPointIndex * 2 + 1);
if (srcVal < destVal)
{
*(destNnfP + destPointIndex * 2 + 0) = *(srcNnfP + destPointIndex * 2 + 0);
*(destNnfP + destPointIndex * 2 + 1) = srcVal;
}
}
else
{
// When the destNnf doesn't contain the value
// for that point we simply copy it
*(destNnfP + destPointIndex * 2 + 0) = *(srcNnfP + destPointIndex * 2 + 0);
*(destNnfP + destPointIndex * 2 + 1) = *(srcNnfP + destPointIndex * 2 + 1);
}
}
}
});
}
/// <summary>
/// Clones the NNF and scales it up 2 times without distances recalculation.
/// </summary>
/// <param name="nnf">The NNF.</param>
/// <param name="scaledDestImage">The scaled dest image.</param>
/// <param name="scaledSrcImage">The scaled source image.</param>
/// <param name="options">The options for parallel processing.</param>
/// <param name="scaledMap">The areas mapping. By default whole area of the dest image is associated with the whole area of the source image.</param>
/// <returns></returns>
/// <exception cref="ArgumentNullException">
/// nnf
/// or
/// scaledDestImage
/// or
/// scaledSrcImage
/// </exception>
public static Nnf CloneAndScaleNnf2X(this Nnf nnf, ZsImage scaledDestImage, ZsImage scaledSrcImage, ParallelOptions options = null, Area2DMap scaledMap = null)
{
if (nnf == null)
{
throw new ArgumentNullException(nameof(nnf));
}
if (scaledDestImage == null)
{
throw new ArgumentNullException(nameof(scaledDestImage));
}
if (scaledSrcImage == null)
{
throw new ArgumentNullException(nameof(scaledSrcImage));
}
if (scaledMap == null)
{
var destArea = Area2D.Create(0, 0, scaledDestImage.Width, scaledDestImage.Height);
var srcArea = Area2D.Create(0, 0, scaledSrcImage.Width, scaledSrcImage.Height);
scaledMap = new Area2DMapBuilder()
.InitNewMap(destArea, srcArea)
.Build();
}
if (options == null)
{
options = new ParallelOptions();
}
var destImageWidth = scaledDestImage.Width;
var srcImageWidth = scaledSrcImage.Width;
var sameSrcAndDest = scaledDestImage == scaledSrcImage;
var mappings = scaledMap.ExtractMappedAreasInfo(destImageWidth, srcImageWidth, true);
var nnf2x = new Nnf(nnf.DstWidth * 2, nnf.DstHeight * 2, nnf.SourceWidth * 2, nnf.SourceHeight * 2, nnf.PatchSize);
var nnfDestWidth = nnf.DstWidth;
var nnfSourceWidth = nnf.SourceWidth;
var nnf2xSourceWidth = nnf2x.SourceWidth;
var nnf2xDstWidth = nnf2x.DstWidth;
var nnfData = nnf.GetNnfItems();
var nnf2xData = nnf2x.GetNnfItems();
// Decide on how many partitions we should divade the processing
// of the elements.
int nnfPointsAmount = nnf.DstWidth * nnf.DstHeight;
var partsCount = nnfPointsAmount > options.NotDividableMinAmountElements
? options.ThreadsCount
: 1;
var partSize = nnfPointsAmount / partsCount;
var offs = new[]
{
new[] { 0, 0 },
new[] { 1, 0 },
new[] { 0, 1 },
new[] { 1, 1 }
};
Parallel.For(0, partsCount, partIndex =>
//for (int partIndex = 0; partIndex < partsCount; partIndex++)
{
bool isPatchFit = false;
var mappedAreasInfos = new MappedAreasInfo[mappings.Length];
for (var i = 0; i < mappings.Length; i++)
{
mappedAreasInfos[i] = mappings[i].Clone();
}
var firstPointIndex = partIndex * partSize;
var lastPointIndex = firstPointIndex + partSize - 1;
if (partIndex == partsCount - 1)
{
lastPointIndex = nnfPointsAmount - 1;
}
if (lastPointIndex > nnfPointsAmount)
{
lastPointIndex = nnfPointsAmount - 1;
}
MappedAreasInfo mappedAreasInfo = null;
for (var j = firstPointIndex; j <= lastPointIndex; j++)
{
var destPx = j;
var destX = destPx % nnfDestWidth;
var destY = destPx / nnfDestWidth;
// Find
var srcPointIndex = nnfData[destY * nnfDestWidth * 2 + destX * 2]; // / 2;
int srcY = (int)(srcPointIndex / nnfSourceWidth); // * 2;
int srcX = (int)(srcPointIndex % nnfSourceWidth); // * 2;
var dist = nnfData[destY * nnfDestWidth * 2 + destX * 2 + 1];
var nY = destY * 2;
var nX = destX * 2;
for (int i = 0; i < offs.Length; i++)
{
var destPointIndex = (nY + offs[i][1]) * nnf2xDstWidth + nX + offs[i][0];
mappedAreasInfo = mappedAreasInfos.FirstOrDefault(mai => mai.DestAreaPointsIndexesSet.Contains(destPointIndex));
if (mappedAreasInfo != null)
{
nnf2xData[destPointIndex * 2] = (srcY * 2 + offs[i][1]) * nnf2xSourceWidth + (srcX * 2 + offs[i][0]);
nnf2xData[destPointIndex * 2 + 1] = dist;
}
else
{
if (sameSrcAndDest)
{
// when the source and the dest image is the same one, the best
// corresponding patch is the patch itself!
nnf2xData[destPointIndex * 2] = destPointIndex;
nnf2xData[destPointIndex * 2 + 1] = 0;
}
else
{
nnf2xData[destPointIndex * 2] = (srcY * 2 + offs[i][1]) * nnf2xSourceWidth + (srcX * 2 + offs[i][0]);
nnf2xData[destPointIndex * 2 + 1] = nnfData[destY * nnfDestWidth * 2 + destX * 2 + 1];
}
}
}
}
}
);
return(nnf2x);
}
/// <summary>
/// Clones the NNF and scales it up 2 times with distances recalculation.
/// </summary>
/// <param name="nnf">The NNF.</param>
/// <param name="scaledDestImage">The scaled dest image.</param>
/// <param name="scaledSrcImage">The scaled source image.</param>
/// <param name="options">The options for parallel processing.</param>
/// <param name="scaledMap">The areas mapping. By default whole area of the dest image is associated with the whole area of the source image.</param>
/// <param name="patchDistanceCalculator">The calculator that calculates similarity of two patches. By deafult the Cie76 is used.</param>
/// <param name="destPixelsArea">Area on the dest image that actually containes pixels. By default is the area of the entire image.</param>
/// <returns></returns>
/// <exception cref="ArgumentNullException">nnf
/// or
/// scaledDestImage
/// or
/// scaledSrcImage
/// or
/// scaledMap</exception>
public static unsafe Nnf CloneAndScale2XWithUpdate(this Nnf nnf, ZsImage scaledDestImage, ZsImage scaledSrcImage, ParallelOptions options, Area2DMap scaledMap, ImagePatchDistanceCalculator patchDistanceCalculator = null, Area2D destPixelsArea = null)
{
if (nnf == null)
{
throw new ArgumentNullException(nameof(nnf));
}
if (scaledDestImage == null)
{
throw new ArgumentNullException(nameof(scaledDestImage));
}
if (scaledSrcImage == null)
{
throw new ArgumentNullException(nameof(scaledSrcImage));
}
if (scaledMap == null)
{
throw new ArgumentNullException(nameof(scaledMap));
}
if (destPixelsArea == null)
{
destPixelsArea = Area2D.Create(0, 0, scaledDestImage.Width, scaledDestImage.Height);
}
if (patchDistanceCalculator == null)
{
patchDistanceCalculator = ImagePatchDistance.Cie76;
}
if (options == null)
{
options = new ParallelOptions();
}
var patchSize = nnf.PatchSize;
var patchLength = patchSize * patchSize;
var destImageWidth = scaledDestImage.Width;
var srcImageWidth = scaledSrcImage.Width;
var sameSrcAndDest = scaledDestImage == scaledSrcImage;
var pixelsArea = (scaledMap as IAreasMapping).DestArea;
//var destPointIndexes = GetDestPointsIndexes(pixelsArea, destImageWidth, NeighboursCheckDirection.Forward);
pixelsArea = pixelsArea.Intersect(destPixelsArea);
var destAvailablePixelsIndexes = pixelsArea.GetPointsIndexes(destImageWidth);
var mappings = scaledMap.ExtractMappedAreasInfo(destImageWidth, srcImageWidth, true);
var nnf2x = new Nnf(nnf.DstWidth * 2, nnf.DstHeight * 2, nnf.SourceWidth * 2, nnf.SourceHeight * 2, nnf.PatchSize);
var nnfDestWidth = nnf.DstWidth;
var nnfSourceWidth = nnf.SourceWidth;
var nnf2xSourceWidth = nnf2x.SourceWidth;
var nnf2xDstWidth = nnf2x.DstWidth;
var nnfData = nnf.GetNnfItems();
var nnf2xData = nnf2x.GetNnfItems();
// Decide on how many partitions we should divade the processing
// of the elements.
int nnfPointsAmount = nnf.DstWidth * nnf.DstHeight;
var partsCount = nnfPointsAmount > options.NotDividableMinAmountElements
? options.ThreadsCount
: 1;
var partSize = nnfPointsAmount / partsCount;
var offs = new[]
{
new[] { 0, 0 },
new[] { 1, 0 },
new[] { 0, 1 },
new[] { 1, 1 }
};
Parallel.For(0, partsCount, partIndex =>
//for (int partIndex = 0; partIndex < partsCount; partIndex++)
{
bool isPatchFit = false;
// Init the dest & source patch
var destPatchPixelsIndexes = new int[patchLength];
var srcPatchPixelsIndexes = new int[patchLength];
//var destPointsIndexesSet = new HashSet<int>(destPointIndexes);
var destAvailablePixelsIndexesSet = new HashSet <int>(destAvailablePixelsIndexes);
var mappedAreasInfos = new MappedAreasInfo[mappings.Length];
for (var i = 0; i < mappings.Length; i++)
{
mappedAreasInfos[i] = mappings[i].Clone();
}
var firstPointIndex = partIndex * partSize;
var lastPointIndex = firstPointIndex + partSize - 1;
if (partIndex == partsCount - 1)
{
lastPointIndex = nnfPointsAmount - 1;
}
if (lastPointIndex > nnfPointsAmount)
{
lastPointIndex = nnfPointsAmount - 1;
}
fixed(double *destImagePixelsDataP = scaledDestImage.PixelsData)
fixed(double *sourceImagePixelsDataP = scaledSrcImage.PixelsData)
fixed(int *srcPatchPixelsIndexesP = srcPatchPixelsIndexes)
fixed(int *destPatchPixelsIndexesP = destPatchPixelsIndexes)
{
MappedAreasInfo mappedAreasInfo = null;
for (var j = firstPointIndex; j <= lastPointIndex; j++)
{
var destPx = j;
var destX = destPx % nnfDestWidth;
var destY = destPx / nnfDestWidth;
// Find
var srcPointIndex = nnfData[destY * nnfDestWidth * 2 + destX * 2]; // / 2;
int srcY = (int)(srcPointIndex / nnfSourceWidth); // * 2;
int srcX = (int)(srcPointIndex % nnfSourceWidth); // * 2;
var nY = destY * 2;
var nX = destX * 2;
for (int i = 0; i < offs.Length; i++)
{
var destPointIndex = (nY + offs[i][1]) * nnf2xDstWidth + nX + offs[i][0];
mappedAreasInfo = mappedAreasInfos.FirstOrDefault(mai => mai.DestAreaPointsIndexesSet.Contains(destPointIndex));
if (mappedAreasInfo != null)
{
nnf2xData[destPointIndex * 2] = (srcY * 2 + offs[i][1]) * nnf2xSourceWidth + (srcX * 2 + offs[i][0]);
Utils.PopulatePatchPixelsIndexes(srcPatchPixelsIndexesP, srcX + offs[i][0], srcY + offs[i][1], patchSize, nnf2xSourceWidth, mappedAreasInfo.SrcAreaPointsIndexesSet, out isPatchFit);
Utils.PopulatePatchPixelsIndexes(destPatchPixelsIndexesP, destX + offs[i][0], destY + offs[i][1], patchSize, destImageWidth, destAvailablePixelsIndexesSet, out isPatchFit);
nnf2xData[destPointIndex * 2 + 1] = patchDistanceCalculator.Calculate(destPatchPixelsIndexesP, srcPatchPixelsIndexesP, double.MaxValue, destImagePixelsDataP, sourceImagePixelsDataP, scaledDestImage, scaledSrcImage, patchLength);
}
else
{
if (sameSrcAndDest)
{
// when the source and the dest image is the same one, the best
// corresponding patch is the patch itself!
nnf2xData[destPointIndex * 2] = destPointIndex;
nnf2xData[destPointIndex * 2 + 1] = 0;
}
else
{
nnf2xData[destPointIndex * 2] = (srcY * 2 + offs[i][1]) * nnf2xSourceWidth + (srcX * 2 + offs[i][0]);
nnf2xData[destPointIndex * 2 + 1] = nnfData[destY * nnfDestWidth * 2 + destX * 2 + 1];
}
}
}
}
}
}
);
return(nnf2x);
}
private InpaintingResult Inpaint(ZsImage image, Area2D removeArea, Nnf nnf, double k, IInpaintSettings settings)
{
// Since the nnf was normalized, the sigma now is normalized as well and it
// has not any more some specific value.
//const double naturalLogBase = System.Math.E;
//const double minusSigmaCube2 = -0.91125;//-2 * sigma * sigma; sigma = 0.675 = 75percentil
//double NaturalLogBase2 = System.Math.Pow(System.Math.E, 1.0 / minusSigmaCube2);
const double naturalLogBasePowMinusSigmaCube2 = 0.33373978049163078;
const double maxSquareDistanceInLab = 32668.1151;
// Gamma is used for calculation of alpha in markup. The confidence.
const double gamma = 1.3;
// The value of confidence in non marked areas.
const double confidentValue = 1.50;
var patchSize = settings.PatchSize;
var colorResolver = settings.ColorResolver;
var pixelChangeTreshold = settings.PixelChangeTreshold;
// Get points' indexes that that needs to be inpainted.
var pointIndexes = new int[removeArea.ElementsCount];
removeArea.FillMappedPointsIndexes(pointIndexes, image.Width);
var patchOffset = (patchSize - 1) / 2;
var imageWidth = image.Width;
var nnfWidth = nnf.DstWidth;
var nnfHeight = nnf.DstHeight;
var srcWidth = nnf.SourceWidth;
var srcHeight = nnf.SourceHeight;
var nnfdata = nnf.GetNnfItems();
var cmpts = image.NumberOfComponents;
// Weighted color is color's components values + weight of the color.
var weightedColors = new double[patchSize * patchSize * (cmpts + 1)];
var confidence = removeArea.CalculatePointsConfidence(confidentValue, gamma);
var resolvedColor = new double[cmpts];
var totalDifference = 0.0;
var changedPixelsAmount = 0;
var changedPixelsDifference = 0.0;
for (int ii = 0; ii < pointIndexes.Length; ii++)
{
var pointIndex = pointIndexes[ii];
int pty = pointIndex / imageWidth;
int ptx = pointIndex % imageWidth;
int colorsCount = 0;
//go thru the patch
for (int y = pty - patchOffset, j = 0; j < patchSize; j++, y++)
{
for (int x = ptx - patchOffset, i = 0; i < patchSize; i++, x++)
{
if (0 <= x && x < nnfWidth && 0 <= y && y < nnfHeight)
{
int destPatchPointIndex = y * nnfWidth + x;
int srcPatchPointIndex = (int)nnfdata[destPatchPointIndex * 2];
int srcPatchPointX = srcPatchPointIndex % nnfWidth;
int srcPatchPointY = srcPatchPointIndex / nnfWidth;
int srcPixelX = srcPatchPointX - i + patchOffset;
int srcPixelY = srcPatchPointY - j + patchOffset;
if (0 <= srcPixelX && srcPixelX < srcWidth && 0 <= srcPixelY && srcPixelY < srcHeight)
{
int srcPixelIndex = srcPixelY * imageWidth + srcPixelX;
for (int ci = 0; ci < cmpts; ci++)
{
weightedColors[colorsCount * (cmpts + 1) + ci] = image.PixelsData[srcPixelIndex * cmpts + ci];
}
weightedColors[colorsCount * (cmpts + 1) + cmpts] =
System.Math.Pow(naturalLogBasePowMinusSigmaCube2, nnfdata[destPatchPointIndex * 2 + 1]) *
confidence[ii];
colorsCount++;
}
}
}
}
// calculate color
colorResolver.Resolve(weightedColors, 0, colorsCount, cmpts, k, resolvedColor, 0);
// how different is resolvedColor from the old color?
var labL = resolvedColor[0] - image.PixelsData[pointIndex * cmpts + 0];
var labA = resolvedColor[1] - image.PixelsData[pointIndex * cmpts + 1];
var labB = resolvedColor[2] - image.PixelsData[pointIndex * cmpts + 2];
var pixelsDiff = (labL * labL + labA * labA + labB * labB) / maxSquareDistanceInLab;
totalDifference += pixelsDiff;
if (pixelsDiff > pixelChangeTreshold)
{
changedPixelsAmount++;
changedPixelsDifference += pixelsDiff;
}
for (var i = 0; i < cmpts; i++)
{
image.PixelsData[pointIndex * cmpts + i] = resolvedColor[i];
}
}
totalDifference /= pointIndexes.Length;
_inpaintingResult.TotalDifference = totalDifference / pointIndexes.Length;
_inpaintingResult.PixelsChangedAmount = changedPixelsAmount;
_inpaintingResult.PixelsToInpaintAmount = pointIndexes.Length;
_inpaintingResult.ChangedPixelsDifference = changedPixelsDifference;
return(_inpaintingResult);
}
/// <summary>
/// Runs the NNF build iteration for the associated areas of the dest and the source images.
/// </summary>
/// <param name="nnf">The NNF.</param>
/// <param name="destImage">The dest image. For each patch at this image we will look for a similar one at the source image.</param>
/// <param name="srcImage">The source image. Source of the patches for the dest image.</param>
/// <param name="direction">The direction to look for a patches.</param>
/// <param name="settings">The settings that control parameters of the algorithm.</param>
/// <param name="patchDistanceCalculator">The calculator that calculates similarity of two patches. By deafult the Cie76 is used.</param>
/// <param name="areasMapping">The areas mapping. By default whole area of the dest image is associated with the whole area of the source image.</param>
/// <param name="destPixelsArea">Area on the dest image that actually containes pixels. By default is the area of the entire image.</param>
/// <exception cref="ArgumentNullException">
/// nnf
/// or
/// destImage
/// or
/// srcImage
/// or
/// settings
/// or
/// patchDistanceCalculator
/// or
/// areasMapping
/// or
/// destPixelsArea
/// </exception>
public unsafe void RunBuildNnfIteration(Nnf nnf, ZsImage destImage, ZsImage srcImage, NeighboursCheckDirection direction, PatchMatchSettings settings, ImagePatchDistanceCalculator patchDistanceCalculator, Area2DMap areasMapping, Area2D destPixelsArea)
{
if (nnf == null)
{
throw new ArgumentNullException(nameof(nnf));
}
if (destImage == null)
{
throw new ArgumentNullException(nameof(destImage));
}
if (srcImage == null)
{
throw new ArgumentNullException(nameof(srcImage));
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (patchDistanceCalculator == null)
{
throw new ArgumentNullException(nameof(patchDistanceCalculator));
}
if (areasMapping == null)
{
throw new ArgumentNullException(nameof(areasMapping));
}
if (destPixelsArea == null)
{
throw new ArgumentNullException(nameof(destPixelsArea));
}
sbyte offs = (sbyte)(direction == NeighboursCheckDirection.Forward ? 1 : -1);
sbyte[][] offsets =
{
new[] { offs, (sbyte)0 },
new[] { (sbyte)0, offs }
};
#region Cache settings and inputs in the local variables
var nnfdata = nnf.GetNnfItems();
var nnfSrcWidth = nnf.SourceWidth;
var patchSize = settings.PatchSize;
var patchLength = patchSize * patchSize;
var randomSearchRadius = settings.RandomSearchRadius;
var alpha = settings.RandomSearchAlpha;
var minSearchWindowSize = settings.MinSearchWindowSize;
var destImageWidth = destImage.Width;
var srcImageWidth = srcImage.Width;
#endregion
// Obtain the data that allow us getting a point index
// regarding to the process direction(forward or backward)
var addModData = GetAddModData(direction, areasMapping.DestElementsCount);
var add = addModData.Item1;
var mod = addModData.Item2;
// Decide on how many partitions we should divade the processing
// of the elements.
var partsCount = areasMapping.DestElementsCount > settings.NotDividableMinAmountElements
? settings.ThreadsCount
: 1;
var partSize = areasMapping.DestElementsCount / partsCount;
var pixelsArea = (areasMapping as IAreasMapping).DestArea;
var destPointIndexes = pixelsArea.GetPointsIndexes(destImageWidth, direction == NeighboursCheckDirection.Forward);
pixelsArea = pixelsArea.Intersect(destPixelsArea);
var destAvailablePixelsIndexes = pixelsArea.GetPointsIndexes(destImageWidth, direction == NeighboursCheckDirection.Forward);
var mappings = areasMapping.ExtractMappedAreasInfo(destImageWidth, srcImageWidth, direction == NeighboursCheckDirection.Forward);
//for (int partIndex = 0; partIndex < partsCount; partIndex++)
Parallel.For(0, partsCount, partIndex =>
{
var rnd = new FastRandom();
int i;
// Colne mapping to avoid conflicts in multithread
//var destPointsIndexesSet = new HashSet<int>(destPointIndexes);
var destAvailablePixelsIndexesSet = new HashSet <int>(destAvailablePixelsIndexes);
var mappedAreasInfos = new MappedAreasInfo[mappings.Length];
for (i = 0; i < mappings.Length; i++)
{
mappedAreasInfos[i] = mappings[i].Clone();
}
var firstPointIndex = partIndex * partSize;
var lastPointIndex = firstPointIndex + partSize - 1;
if (partIndex == partsCount - 1)
{
lastPointIndex = destPointIndexes.Length - 1;
}
if (lastPointIndex > destPointIndexes.Length)
{
lastPointIndex = destPointIndexes.Length - 1;
}
// Init the dest & source patch
var destPatchPixelsIndexes = new int[patchSize * patchSize];
var srcPatchPixelsIndexes = new int[patchSize * patchSize];
bool isPatchFit = false;
fixed(double *nnfP = nnfdata)
fixed(int *destPointIndexesP = destPointIndexes)
fixed(int *srcPatchPixelsIndexesP = srcPatchPixelsIndexes)
fixed(int *destPatchPixelsIndexesP = destPatchPixelsIndexes)
fixed(double *destImagePixelsDataP = destImage.PixelsData)
fixed(double *sourceImagePixelsDataP = srcImage.PixelsData)
{
for (var j = firstPointIndex; j <= lastPointIndex; j++)
{
// Obtain a destination point from the mapping.
// The dest point is the position of the dest patch.
// Chocie of the destination point depends on the
// direction in which we iterate the points (forward or backward).
var destPointIndex = *(destPointIndexesP + add + j * mod);
var destPointX = destPointIndex % destImageWidth;
var destPointY = destPointIndex / destImageWidth;
// We going to find the most similar source patch to the dest patch
// by comparing them.
// Note: we don't care about the coverage state of the dest patch
// because it is at least partially covered. The situation when it is
// not covered at all is impossible, since the dest point is a part of the
// dest area.
// We store our patch in a flat array of points for simplicity.
// Populate dest patch array with corresponding dest image points indexes.
Utils.PopulatePatchPixelsIndexes(destPatchPixelsIndexesP, destPointX, destPointY, patchSize, destImageWidth, destAvailablePixelsIndexesSet, out isPatchFit);
// Obtain the mapped areas info with the destination area
// that contains the current dest point. In the associated
// source area we are allowed to look for the source patches.
MappedAreasInfo mappedAreasInfo = mappedAreasInfos.FirstOrDefault(mai => mai.DestAreaPointsIndexesSet.Contains(destPointIndex));
// To improve performance, we cache the value of the NNF
// at the current point in the local variables.
var nnfPos = destPointIndex * 2;
var bestSrcPointIndex = (int)*(nnfP + nnfPos + 0);
var distanceToBestSrcPatch = *(nnfP + nnfPos + 1);
// We assume that the value of the NNF at the current
// point is the best so far and we have not found anything beter.
var isBetterSrcPatchFound = false;
// Now we check two close neighbours. First - horisontal, than vertical.
#region f(x - p) + (1, 0) & f(x - p) + (0, 1)
//1st iteration: f(x - p) + (1, 0)
//2nd iteration: f(x - p) + (0, 1)
int candidateSrcPointY = 0;
int candidateSrcPointX = 0;
double distance;
for (var offsetIndex = 0; offsetIndex < offsets.Length; offsetIndex++)
{
// Obtain the position of the neighbour.
// During the forward search
// on the first iteration it is the neighbour at the left side
// on the second iteration it is the neighbour at the top
// During the backward search
// on the first iteration it is the neighbour at the right side
// on the second iteration it is the neighbour at the bottom
var offset = offsets[offsetIndex];
var destPointNeighbourX = destPointX - offset[0];
var destPointNeighbourY = destPointY - offset[1];
var destPointNeighbourIndex = destPointNeighbourY * destImageWidth + destPointNeighbourX;
// Make sure that the neighbour point belongs to the mapping
if (!mappedAreasInfo.DestAreaPointsIndexesSet.Contains(destPointNeighbourIndex))
{
continue;
}
// There is a high chance that the neighbour's source patch neighbour
// is similar to the current dest patch. Let's check it.
// Obtain the position of the neighbour's source patch neighbour.
var neighbourNnfPos = 2 * destPointNeighbourIndex;
var neighbourSrcPointIndex = (int)*(nnfP + neighbourNnfPos + 0);
candidateSrcPointY = neighbourSrcPointIndex / nnfSrcWidth + offset[1];
candidateSrcPointX = neighbourSrcPointIndex % nnfSrcWidth + offset[0];
var candidateSrcPointIndex = candidateSrcPointY * srcImageWidth + candidateSrcPointX;
// Make sure that the source patch resides inside the
// associated source area of the dest area.
if (!mappedAreasInfo.SrcAreaPointsIndexesSet.Contains(candidateSrcPointIndex))
{
continue;
}
// Populate source patch array with corresponding source image points indexes
Utils.PopulatePatchPixelsIndexes(srcPatchPixelsIndexesP, candidateSrcPointX, candidateSrcPointY, patchSize, srcImageWidth, mappedAreasInfo.SrcAreaPointsIndexesSet, out isPatchFit);
// Calculate distance between the patches
distance = patchDistanceCalculator.Calculate(destPatchPixelsIndexesP, srcPatchPixelsIndexesP, distanceToBestSrcPatch, destImagePixelsDataP, sourceImagePixelsDataP, destImage, srcImage, patchLength);
if (isPatchFit && distance < distanceToBestSrcPatch)
{
isBetterSrcPatchFound = true;
bestSrcPointIndex = candidateSrcPointIndex;
distanceToBestSrcPatch = distance;
}
}
#endregion
#region random search
//v0 = f(x)
// For the random search we need to find out
// the size of the window where to search
var srcAreaBound = mappedAreasInfo.SrcBound;
var srcAreaWidth = srcAreaBound.Width;
var srcAreaHeight = srcAreaBound.Height;
var searchWindowSize = srcAreaHeight < srcAreaWidth ? srcAreaHeight : srcAreaWidth;
if (searchWindowSize > minSearchWindowSize)
{
searchWindowSize = minSearchWindowSize;
}
// Init the search radius;
var searchRadius = randomSearchRadius;
var bestSrcPointX = bestSrcPointIndex % srcImageWidth;
var bestSrcPointY = bestSrcPointIndex / srcImageWidth;
for (i = 0; searchRadius >= 1; i++)
{
//TODO: change and measure inpact of changing call to Pow
searchRadius = searchWindowSize * System.Math.Pow(alpha, i);
isPatchFit = false;
var attempts = 0;
while (!isPatchFit && attempts < 5)
{
attempts++;
candidateSrcPointX = (int)(bestSrcPointX + (rnd.NextDouble() * 2 - 1.0) * searchRadius);
candidateSrcPointY = (int)(bestSrcPointY + (rnd.NextDouble() * 2 - 1.0) * searchRadius);
// Populate source patch array with corresponding source image points indexes
Utils.PopulatePatchPixelsIndexes(srcPatchPixelsIndexesP, candidateSrcPointX, candidateSrcPointY, patchSize, srcImageWidth, mappedAreasInfo.SrcAreaPointsIndexesSet, out isPatchFit);
}
// Calculate distance between the patches
distance = patchDistanceCalculator.Calculate(destPatchPixelsIndexesP, srcPatchPixelsIndexesP, distanceToBestSrcPatch, destImagePixelsDataP, sourceImagePixelsDataP, destImage, srcImage, patchLength);
if (isPatchFit && distance < distanceToBestSrcPatch)
{
distanceToBestSrcPatch = distance;
bestSrcPointIndex = candidateSrcPointY * srcImageWidth + candidateSrcPointX;
isBetterSrcPatchFound = true;
}
}
#endregion
if (isBetterSrcPatchFound)
{
*(nnfP + nnfPos + 0) = bestSrcPointIndex;
*(nnfP + nnfPos + 1) = distanceToBestSrcPatch;
}
}
}
});
}
/// <summary>
/// Runs the random NNF initialization iteration for the associated areas of the dest and the source images.
/// </summary>
/// <param name="nnf">The NNF.</param>
/// <param name="destImage">The dest image. For each patch at this image we will look for a similar one at the source image.</param>
/// <param name="srcImage">The source image. Source of the patches for the dest image.</param>
/// <param name="settings">The settings that control parameters of the algorithm.</param>
/// <param name="patchDistanceCalculator">The calculator that calculates similarity of two patches. By deafult the Cie76 is used.</param>
/// <param name="areasMapping">The areas mapping. By default whole area of the dest image is associated with the whole area of the source image.</param>
/// <param name="destPixelsArea">Area on the dest image that actually containes pixels. By default is the area of the entire image.</param>
/// <exception cref="ArgumentNullException">
/// nnf
/// or
/// destImage
/// or
/// srcImage
/// or
/// settings
/// or
/// patchDistanceCalculator
/// or
/// areasMapping
/// or
/// destPixelsArea
/// </exception>
public unsafe void RunRandomNnfInitIteration(Nnf nnf, ZsImage destImage, ZsImage srcImage, PatchMatchSettings settings, ImagePatchDistanceCalculator patchDistanceCalculator, Area2DMap areasMapping, Area2D destPixelsArea)
{
if (nnf == null)
{
throw new ArgumentNullException(nameof(nnf));
}
if (destImage == null)
{
throw new ArgumentNullException(nameof(destImage));
}
if (srcImage == null)
{
throw new ArgumentNullException(nameof(srcImage));
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (patchDistanceCalculator == null)
{
throw new ArgumentNullException(nameof(patchDistanceCalculator));
}
if (areasMapping == null)
{
throw new ArgumentNullException(nameof(areasMapping));
}
if (destPixelsArea == null)
{
throw new ArgumentNullException(nameof(destPixelsArea));
}
const byte MaxAttempts = 32;
var nnfdata = nnf.GetNnfItems();
var patchSize = settings.PatchSize;
var patchPointsAmount = patchSize * patchSize;
var destImageWidth = destImage.Width;
var srcImageWidth = srcImage.Width;
// Decide on how many partitions we should divade the processing
// of the elements.
var partsCount = areasMapping.DestElementsCount > settings.NotDividableMinAmountElements
? settings.ThreadsCount
: 1;
var partSize = (int)(areasMapping.DestElementsCount / partsCount);
var pixelsArea = (areasMapping as IAreasMapping).DestArea;
var destPointsIndexes = pixelsArea.GetPointsIndexes(destImageWidth);
pixelsArea = pixelsArea.Intersect(destPixelsArea);
var destAvailablePixelsIndexes = pixelsArea.GetPointsIndexes(destImageWidth);
var mappings = areasMapping.ExtractMappedAreasInfo(destImageWidth, srcImageWidth);
//for (int partIndex = 0; partIndex < partsCount; partIndex++)
Parallel.For(0, partsCount, partIndex =>
{
// Colne mapping to avoid conflicts in multithread
//var destPointsIndexesSet = new HashSet<int>(destPointsIndexes);
var destAvailablePixelsIndexesSet = new HashSet <int>(destAvailablePixelsIndexes);
var mappedAreasInfos = new MappedAreasInfo[mappings.Length];
for (int i = 0; i < mappings.Length; i++)
{
mappedAreasInfos[i] = mappings[i].Clone();
}
#region Variables definition
int destPointIndex;
int destPointX;
int destPointY;
int srcPointIndex = 0;
int srcPointX;
int srcPointY;
double distance;
int nnfPos;
byte attemptsCount = 0;
#endregion
var rnd = new FastRandom();
var firstPointIndex = partIndex * partSize;
var lastPointIndex = firstPointIndex + partSize - 1;
if (partIndex == partsCount - 1)
{
lastPointIndex = areasMapping.DestElementsCount - 1;
}
if (lastPointIndex > areasMapping.DestElementsCount)
{
lastPointIndex = areasMapping.DestElementsCount - 1;
}
// Init the dest & source patch
var destPatchPointIndexes = new int[patchPointsAmount];
var srcPatchPointIndexes = new int[patchPointsAmount];
bool isPatchFit = false;
fixed(double *nnfdataP = nnfdata)
fixed(int *dstPointIndexesP = destPointsIndexes)
fixed(int *srcPatchPointIndexesP = srcPatchPointIndexes)
fixed(int *destPatchPointIndexesP = destPatchPointIndexes)
fixed(double *destPixelsDataP = destImage.PixelsData)
fixed(double *sourcePixelsDataP = srcImage.PixelsData)
{
for (var pointIndex = firstPointIndex; pointIndex <= lastPointIndex; pointIndex++)
{
destPointIndex = *(dstPointIndexesP + pointIndex);
destPointX = destPointIndex % destImageWidth;
destPointY = destPointIndex / destImageWidth;
Utils.PopulatePatchPixelsIndexes(destPatchPointIndexesP, destPointX, destPointY, patchSize, destImageWidth, destAvailablePixelsIndexesSet, out isPatchFit);
nnfPos = destPointIndex * 2;
// Obtain the source area associated with the destination area
// to which the current dest point belongs to.
// In that area we are allowed to look for the source patches.
MappedAreasInfo mappedAreasInfo = mappedAreasInfos.FirstOrDefault(mai => mai.DestAreaPointsIndexesSet.Contains(destPointIndex));
isPatchFit = false;
attemptsCount = 0;
var srcPointsAmount = mappedAreasInfo.SrcAreaPointsIndexes.Length;
while (!isPatchFit && attemptsCount < MaxAttempts && attemptsCount < srcPointsAmount)
{
srcPointIndex = mappedAreasInfo.SrcAreaPointsIndexes[rnd.Next(srcPointsAmount)];
srcPointX = srcPointIndex % srcImageWidth;
srcPointY = srcPointIndex / srcImageWidth;
Utils.PopulatePatchPixelsIndexes(srcPatchPointIndexesP, srcPointX, srcPointY, patchSize, srcImageWidth, mappedAreasInfo.SrcAreaPointsIndexesSet, out isPatchFit);
attemptsCount++;
}
distance = patchDistanceCalculator.Calculate(destPatchPointIndexesP, srcPatchPointIndexesP, double.MaxValue,
destPixelsDataP, sourcePixelsDataP, destImage, srcImage, patchPointsAmount);
*(nnfdataP + nnfPos + 0) = srcPointIndex;
*(nnfdataP + nnfPos + 1) = distance;
}
}
});
}
