/// <summary>
/// Find brightest red pixel in a masked image
/// Use Laplacian filter to detect areas where the red color component has changed more than blue or green
/// </summary>
/// <param name="bgraMat">Image data in BGRA format</param>
/// <param name="mask">mask of relavant areas to search</param>
/// <param name="originalWidth">width of the original (non-cropped) image used for parameter tuning</param>
private Point FindBrightestPoint(Mat bgraMat, Mat mask, int originalWidth)
{
Mat channel = new Mat(bgraMat.height(), bgraMat.width(), CvType.CV_8UC1);
Mat blueContrast = new Mat(bgraMat.height(), bgraMat.width(), CvType.CV_16SC1);
Mat greenContrast = new Mat(bgraMat.height(), bgraMat.width(), CvType.CV_16SC1);
Mat redContrast = new Mat(bgraMat.height(), bgraMat.width(), CvType.CV_16SC1);
Mat sum = new Mat(bgraMat.height(), bgraMat.width(), CvType.CV_16SC1);
var kernelSize = originalWidth > 1600 ? 5 : 3;
Core.extractChannel(bgraMat, channel, 0);
Imgproc.Laplacian(channel, blueContrast, CvType.CV_16S, kernelSize, 1, 0, Core.BORDER_REPLICATE); // calculate contrast in blue channel to detect sudden increase of red component
Core.extractChannel(bgraMat, channel, 1);
Imgproc.Laplacian(channel, greenContrast, CvType.CV_16S, kernelSize, 1, 0, Core.BORDER_REPLICATE); // calculate contrast in green channel to detect sudden increase of red component
Core.extractChannel(bgraMat, channel, 2);
Imgproc.Laplacian(channel, redContrast, CvType.CV_16S, kernelSize, 1, 0, Core.BORDER_REPLICATE); // calculate contrast in red channel to detect sudden increase of red component
channel.convertTo(sum, CvType.CV_16S);
sum *= 2; // give absolute brightness of red component double weight
sum -= 3 * redContrast; // boost weight of pixels with a negative red curvature, i.e. whose neighbors are less red.
sum += blueContrast; // oposite weights for changes in blue and green contrast, to give pixels a stronger boost whose red component changed but blue and green remained constant
sum += greenContrast;
var minMax = Core.minMaxLoc(sum, mask); // get pixel with strongest contrast towards red among reddish things
sum.Dispose();
redContrast.Dispose();
greenContrast.Dispose();
blueContrast.Dispose();
channel.Dispose();
return(minMax.maxLoc);
}
void ComputerVisionAlgo(IntPtr greyscale)
{
Utils.copyToMat(greyscale, imageMat);
Imgproc.GaussianBlur(imageMat, gaussMat, GAUSS_SIZE, GAUSS_SIGMA, GAUSS_SIGMA);
Imgproc.Laplacian(gaussMat, logMat, CvType.CV_8UC1, 5);
logMat = logMat * GAUSS_SIGMA;
outMat = logMat;
}
void domainTransferDepthImage(Frame3DGPU f)
{
//Utils.setDebugMode(true);
Debug.Log("Applying EdgeCleanup to Depth");
// convert from texture to mat
Mat rgbMat = new Mat();
Core.flip(Util.toMat(f.postprocessedRGBImage, CvType.CV_8UC3), rgbMat, -1);
Mat depthMat = Util.toMat(f.depthImage, CvType.CV_16UC1);
Mat gray = new Mat();
Imgproc.cvtColor(rgbMat, gray, Imgproc.COLOR_RGBA2GRAY);
Mat sobelX = new Mat();
Mat sobelY = new Mat();
Imgproc.Sobel(gray, sobelX, CvType.CV_16S, 1, 0, (int)ksize, sobelScale, 0, Core.BORDER_DEFAULT);
Imgproc.Sobel(gray, sobelY, CvType.CV_16S, 0, 1, (int)ksize, sobelScale, 0, Core.BORDER_DEFAULT);
Mat depthMat8bit = new Mat();
depthMat.convertTo(depthMat8bit, CvType.CV_8UC1, 0.03f);
Core.bitwise_not(depthMat8bit, depthMat8bit);
//Imgproc.equalizeHist(depthMat8bit, depthMat8bit);
Mat depthFlipped = new Mat();
Core.flip(depthMat8bit, depthFlipped, -1);
Mat canneyRslt = new Mat();
Imgproc.Canny(sobelX, sobelY, canneyRslt, cannyThreshold1, cannyThreshold2, true);
//Imgcodecs.imwrite("C:/Users/SIGLab/AppData/LocalLow/Intel/Photo3D/3dImages/" + "depth.png", canneyRslt);
//415 incomplete depth
Mat cropped = depthFlipped.submat(0, 690, 0, 1190);
Core.copyMakeBorder(cropped, depthFlipped, 0, 720 - 690, 0, 1280 - 1190, Core.BORDER_REPLICATE | Core.BORDER_ISOLATED);
Mat laplacianRslt = new Mat();
Imgproc.Laplacian(gray, laplacianRslt, CvType.CV_32F, 5, .1, 0);
Ximgproc.dtFilter(canneyRslt, depthFlipped, f.refinedDepth, sigmaSpacial, sigmaColor, Ximgproc.DTF_NC, dtIter);
// Not working with built solutions, cant figure out why
List <Mat> matList = new List <Mat>();
Mat depthLUT = Util.toMat(depthRescale, CvType.CV_8UC3);
Core.split(depthLUT, matList);
Mat temp = new Mat();
f.refinedDepth.convertTo(temp, CvType.CV_8UC1);
Core.LUT(temp, matList[0], f.refinedDepth);
//Utils.setDebugMode(false);
}
// Update is called once per frame
void Update()
{
Debug.Log("Applying EdgeCleanup to Depth");
Mat gray = new Mat();
Imgproc.cvtColor(color, gray, Imgproc.COLOR_RGBA2GRAY);
Mat sobelX = new Mat();
Mat sobelY = new Mat();
Imgproc.Sobel(gray, sobelX, CvType.CV_16S, 1, 0, (int)ksize, sobelScale, 0, Core.BORDER_DEFAULT);
Imgproc.Sobel(gray, sobelY, CvType.CV_16S, 0, 1, (int)ksize, sobelScale, 0, Core.BORDER_DEFAULT);
Mat depth = Util.toMat((Texture2D)depthTexture, CvType.CV_8UC3);
Mat depthFlipped = new Mat();
//Core.flip(depthMat, depthFlipped, -1);
Mat depthMat8bit = new Mat();
depth.convertTo(depthMat8bit, CvType.CV_8UC1, 0.1f);
//Core.bitwise_not(depthMat8bit,depthMat8bit);
//Imgproc.equalizeHist(depthMat8bit, depthMat8bit);
Mat canneyRslt = new Mat();
Imgproc.Canny(sobelX, sobelY, canneyRslt, cannyThreshold1, cannyThreshold2, true);
Mat laplacianRslt = new Mat();
Imgproc.Laplacian(gray, laplacianRslt, CvType.CV_32F, 5, .1, 0);
Mat DTF_NC = new Mat();
Ximgproc.dtFilter(canneyRslt, depthMat8bit, DTF_NC, sigmaSpacial, sigmaColor, Ximgproc.DTF_NC, dtIter);
Texture2D yTexture = (Texture2D)Util.toTexture(sobelX, TextureFormat.R16);
sobelTextureEvent.Invoke(yTexture);
Texture2D canneyTexture = (Texture2D)Util.toTexture(canneyRslt, TextureFormat.R8);
canneyTextureEvent.Invoke(canneyTexture);
Texture2D depthtexture = (Texture2D)Util.toTexture(DTF_NC, TextureFormat.R8);
improvedDepth.Invoke(depthtexture);
}
//拉普拉斯滤波
public Sprite LaplaceGradient()
{
Mat dst = new Mat();
int scale = 1;
int delta = 0;
int ddepth = CvType.CV_16S;
int kernel_size = 3;
Imgproc.Laplacian(grayMat, dst, ddepth, kernel_size, scale, delta, Core.BORDER_DEFAULT);
Core.convertScaleAbs(dst, dstMat);
// Mat转Texture2D
Texture2D t2d = new Texture2D(dstMat.cols(), dstMat.rows());
Utils.matToTexture2D(dstMat, t2d);
Sprite sp = Sprite.Create(t2d, new UnityEngine.Rect(0, 0, t2d.width, t2d.height), Vector2.zero);
return(sp);
}
