public Region createRefienedRegion(Mat channel, int threshold)
{
// もともとのマスクを拡張して、追加のチャンネルで閾値処理をする.
Mat newMask = Mat.zeros(parentMat.size(), CvType.CV_8U);
Mat orignalMask = mask.clone();
Imgproc.dilate(orignalMask, orignalMask, Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(30, 30)));
Core.add(newMask, orignalMask, newMask);
Mat c = Mat.zeros(parentMat.size(), CvType.CV_8UC1);
channel.copyTo(c, newMask);
Imgproc.threshold(c, c, threshold, 255, Imgproc.THRESH_BINARY);
var refinedContours = ARUtil.findContours(c);
var refinedCandidate = new RegionCandidateSet(refinedContours).selectWithMaxArea();
if (refinedCandidate == null)
{
return(null);
}
return(new Region(refinedCandidate, parentMat));
}
// Update is called once per frame
void Process()
{
binaryMatCreator.setCrUpper(cr_threshold_upper);
binaryMatCreator.setCrLower(cr_threshold_lower);
binaryMatCreator.setSUpper(s_threshold_upper);
binaryMatCreator.setSLower(s_threshold_lower);
binaryMatCreator.setVUpper(v_threshold_upper);
binaryMatCreator.setVLower(v_threshold_lower);
// UnityのTexture2DからOpencvのMatに変換
// UnityのTexture2DからOpencvのMatに変換
int imageWidth = cameraTexture.width;
int imageHeight = cameraTexture.height;
UnityEngine.Rect wholeRect = new UnityEngine.Rect(0, 0, cameraTexture.width, cameraTexture.height);
cameraTexture.ReadPixels(wholeRect, 0, 0, true);
//int imageWidth = image.Width;
//int imageHeight = image.Height;
//cameraMat = new Mat(imageHeight, imageWidth, CvType.CV_8UC3);
//cameraTexture = new Texture2D(imageWidth, imageHeight, TextureFormat.ARGB32, false);
//image.CopyToTexture(cameraTexture);
Utils.texture2DToMat(cameraTexture, cameraMat);
//var hsvChs = ARUtil.getHSVChannels(cameraMat);
//var yCrCbChs = ARUtil.getYCrCbChannels(cameraMat);
//var sBy = new Mat(cameraMat.size(), CvType.CV_8UC1);
//var crBy = new Mat(cameraMat.size(), CvType.CV_8UC1);
//var vBy = new Mat(cameraMat.size(), CvType.CV_8UC1);
//Core.inRange(hsvChs[1], new Scalar(s_threshold_lower), new Scalar(s_threshold_upper), sBy);
//Core.inRange(yCrCbChs[1], new Scalar(cr_threshold_lower), new Scalar(cr_threshold_upper), crBy);
//Core.inRange(hsvChs[2], new Scalar(v_threshold_lower), new Scalar(v_threshold_upper), vBy);
//camQuad2.setMat(sBy);
//camQuad3.setMat(crBy);
//camQuad4.setMat(vBy);
//goto show;
/* 初期化 */
IScorer scorer = null; // 領域判定のスコアラー
OpenCVForUnity.Rect searchRect = null; // 探索領域矩形
// Vuforiaでのコップ検出可否によって、以下を切り替える
// ・探索対象領域
// ・スコアー戦略
// ・二値化の閾値
var circularity = 0.0;
if (cup.isNotFound())
{
// TODO: 高速化のため、対象物の探索領域を前回領域に限定する
//if (coffeeRegion.circularity > 0.50) {
// searchRect = coffeeRegion.predictNextSearchRect ();
// searchRect = ARUtil.calcRectWithinMat (searchRect, cameraMat);
//} else {
// searchRect = new OpenCVForUnity.Rect (0, 0, cameraMat.cols (), cameraMat.rows ());
//}
searchRect = new OpenCVForUnity.Rect(0, 0, cameraMat.cols(), cameraMat.rows());
circularity = 0.2;
}
else if (cup.isTracked())
{
print("Cup is tracked.");
cup.update();
// カップの上面の矩形を探索対象矩形とする.
searchRect = cup.getTopSurfaceRect(cameraMat);
circularity = 0.1;
}
scorer = new OrangeScorer(searchRect, foodRegion.rect);
try
{
// カメラ入力画像から, searchRectサイズの二値画像を生成
Mat binaryROIMat = binaryMatCreator.createBinaryMat(cameraMat, searchRect);
// 二値画像&探索領域矩形で輪郭探索
var contours = ARUtil.findContours(binaryROIMat, searchRect.tl());
// 飲料領域候補群を作成 -> 円形度で除外 -> 候補にスコアをつける -> 最大スコアの候補を取得
var regionSet = new RegionCandidateSet(contours)
.elliminateByArea(searchRect, 0.01, 0.9)
.elliminateByCircularity(circularity)
.score(scorer)
.sort();
var count = 0;
var regions = new List <Region>();
foreach (var candidate in regionSet.candidates)
{
if (count > 2)
{
break;
}
if (candidate == null)
{
print("candite is null");
break;
}
if (candidate.circularity < 0.4)
{
print("candidate circularity is not enougph");
break;
}
// 領域作成
foodRegion = new Region(candidate, cameraMat);
regions.Add(foodRegion);
}
// Regionのマスクを合体
Mat alphaMask = Mat.zeros(cameraMat.size(), CvType.CV_8U);
foreach (var region in regions)
{
Core.add(region.mask, alphaMask, alphaMask);
}
// テクスチャ作成
texture = _textureCreator.create(cameraMat, alphaMask);
if (willChange)
{
// アルファブレンド
ARUtil.alphaBlend(cameraMat, texture, alpha, alphaMask);
}
//Debug.Log (candidate.circularity);
// 矩形描画
if (shouldDrawRect)
{
Imgproc.rectangle(cameraMat, foodRegion.rect.tl(), foodRegion.rect.br(), new Scalar(0, 255, 0), 3);
}
}
catch (System.Exception e)
{
print(e);
goto show;
}
show:
outputScreenQuad.setMat(cameraMat);
}
void _Process()
{
binaryMatCreator = new BinaryMatCreator();
binaryMatCreator.setCrUpper(cr_threshold_upper);
binaryMatCreator.setCrLower(cr_threshold_lower);
binaryMatCreator.setSUpper(s_threshold_upper);
binaryMatCreator.setSLower(s_threshold_lower);
binaryMatCreator.setVUpper(v_threshold_upper);
binaryMatCreator.setVLower(v_threshold_lower);
Utils.webCamTextureToMat(webCamTexture, rgbaMat, colors);
Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB);
int imageWidth = Screen.width;
int imageHeight = Screen.height;
Mat cameraMat = new Mat(new Size(imageWidth, imageHeight), CvType.CV_8UC3);
Imgproc.resize(rgbMat, cameraMat, cameraMat.size());
//Mat cameraMat = new Mat(rgbaMat.size(), CvType.CV_8UC3);
//Imgproc.cvtColor(rgbaMat, cameraMat, Imgproc.COLOR_RGBA2RGB);
//var hsvChs = ARUtil.getHSVChannels(cameraMat);
//var yCrCbChs = ARUtil.getYCrCbChannels(cameraMat);
//var sBy = new Mat(cameraMat.size(), CvType.CV_8UC1);
//var crBy = new Mat(cameraMat.size(), CvType.CV_8UC1);
//var vBy = new Mat(cameraMat.size(), CvType.CV_8UC1);
//Core.inRange(hsvChs[1], new Scalar(s_threshold_lower), new Scalar(s_threshold_upper), sBy);
//Core.inRange(yCrCbChs[1], new Scalar(cr_threshold_lower), new Scalar(cr_threshold_upper), crBy);
//Core.inRange(hsvChs[2], new Scalar(v_threshold_lower), new Scalar(v_threshold_upper), vBy);
//camQuad2.setMat(sBy);
//camQuad3.setMat(crBy);
//camQuad4.setMat(vBy);
/* 初期化 */
// 探索領域矩形
OpenCVForUnity.Rect searchRect = new OpenCVForUnity.Rect(0, 0, cameraMat.cols(), cameraMat.rows());
// 領域判定のスコアラー
IScorer scorer = null;
if (loopCount == 0)
{
scorer = new OrangeScorer(searchRect, searchRect);
}
else
{
scorer = new OrangeScorer(searchRect, foodRegion.rect);
}
try
{
// カメラ入力画像から, searchRectサイズの二値画像を生成
Mat binaryROIMat = binaryMatCreator.createBinaryMat(cameraMat, searchRect);
// 二値画像&探索領域矩形で輪郭探索
var contours = ARUtil.findContours(binaryROIMat, searchRect.tl());
// 飲料領域候補群を作成 -> 円形度で除外 -> 候補にスコアをつける -> 最大スコアの候補を取得
var regionSet = new RegionCandidateSet(contours)
.elliminateByArea(searchRect, 0.01, 0.9)
.elliminateByCircularity(0.2)
.score(scorer)
.sort();
var count = 0;
var regions = new List <Region>();
foreach (var candidate in regionSet.candidates)
{
if (count > 2)
{
break;
}
if (candidate == null)
{
print("candite is null");
break;
}
// 領域作成
foodRegion = new Region(candidate, cameraMat);
regions.Add(foodRegion);
count++;
}
// Regionのマスクを合体
Mat alphaMask = Mat.zeros(cameraMat.size(), CvType.CV_8U);
foreach (var region in regions)
{
Core.add(region.mask, alphaMask, alphaMask);
}
// テクスチャ作成
var decorateTextureMat = _textureCreator.create(cameraMat, alphaMask, H_sourceMean, 0, 0);
if (willChange)
{
// アルファブレンド
ARUtil.alphaBlend(cameraMat, decorateTextureMat, alpha, alphaMask);
}
if (shouldDrawRect)
{
// 検出領域を矩形描画
Imgproc.rectangle(cameraMat, foodRegion.rect.tl(), foodRegion.rect.br(), new Scalar(0, 255, 0), 3);
}
}
catch (System.Exception e)
{
print(e);
goto show;
}
show:
outputScreenQuad.setMat(cameraMat);
loopCount++;
}
void _Process()
{
binaryMatCreator = new BinaryMatCreator();
binaryMatCreator.setCrUpper(cr_threshold_upper);
binaryMatCreator.setCrLower(cr_threshold_lower);
binaryMatCreator.setSUpper(s_threshold_upper);
binaryMatCreator.setSLower(s_threshold_lower);
binaryMatCreator.setVUpper(v_threshold_upper);
binaryMatCreator.setVLower(v_threshold_lower);
Utils.webCamTextureToMat(webCamTexture, rgbaMat, colors);
Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB);
Mat cameraMat = rgbMat;
int imageWidth = cameraMat.width();
int imageHeight = cameraMat.height();
var hSVChannels = ARUtil.getHSVChannels(cameraMat);
var yCrCbChannels = ARUtil.getYCrCbChannels(cameraMat);
OpenCVForUnity.Rect searchRect = new OpenCVForUnity.Rect(0, 0, cameraMat.cols(), cameraMat.rows());
// 領域判定のスコア
scorer = new MaguroScorer(searchRect);
try
{
// カメラ入力画像から, searchRectサイズの二値画像を生成
Mat binaryROIMat = binaryMatCreator.createBinaryMat(cameraMat, searchRect);
// 二値画像&探索領域矩形で輪郭探索
var contours = ARUtil.findContours(binaryROIMat, searchRect.tl());
// 領域候補群を作成 -> 候補にスコアをつける -> 最大スコアの候補を取得
var regionSet = new RegionCandidateSet(contours)
.elliminateByArea(searchRect, 0.01, 0.9)
.score(scorer)
.sort();
if (regionSet.candidates.Count == 0)
{
print("first candidates is 0");
goto show;
}
var count = 0;
var regions = new List <Region>();
foreach (var candidate in regionSet.candidates)
{
if (count > 2)
{
break;
}
if (candidate == null)
{
print("candite is null");
break;
}
// 領域作成
foodRegion = new Region(candidate, cameraMat);
regions.Add(foodRegion);
count++;
}
//var regions = new List<Region>();
//var cr_refined_threshold = 125;
//foreach (var candidate in regionSet.candidates)
//{
// // 領域作成
// var region = new Region(candidate, cameraMat);
// var refinedRegion = region.createRefienedRegion(yCrCbChannels[1], cr_refined_threshold);
// if (refinedRegion != null)
// {
// regions.Add(refinedRegion);
// }
//}
//var filteredRegions = Region.elliminateByInclusionRect(regions);
// 食品領域に貼りつけるテクスチャ作成
var texture = _textureCreator.create(cameraMat, regions);
if (texture == null)
{
print("regions is empty");
goto show;
}
Mat alphaMask = Mat.zeros(cameraMat.size(), CvType.CV_8U);
foreach (var region in regions)
{
Core.add(region.mask, alphaMask, alphaMask);
}
if (willChange)
{
_textureCreator.alphaBlend(cameraMat, texture, alphaMask, alpha);
}
if (shouldDrawRect)
{
// foodRegion.drawRect(matForDeveloper);
Imgproc.rectangle(cameraMat, searchRect.tl(), searchRect.br(), new Scalar(0, 0, 255), 3);
}
}
catch (System.Exception e)
{
print(e);
goto show;
}
show:
outputScreenQuad.setMat(cameraMat);
}
// Update is called once per frame
void OnPostRender()
{
// UnityのTexture2DからOpencvのMatに変換
UnityEngine.Rect wholeRect = new UnityEngine.Rect(0, 0, cameraTexture.width, cameraTexture.height);
cameraTexture.ReadPixels(wholeRect, 0, 0, true);
Utils.texture2DToMat(cameraTexture, cameraMat);
/* 初期化 */
IScorer scorer = null; // 領域判定のスコアラー
OpenCVForUnity.Rect searchRect = null; // 探索領域矩形
double v_threshold = 0.0; // 輪郭探索するための二値画像を作るときの明るさの閾値
// Vuforiaでのコップ検出可否によって、以下を切り替える
// ・探索対象領域
// ・スコアー戦略
// ・二値化の閾値
if (cup.isNotFound())
{
// TODO: 高速化のため、対象物の探索領域を前回領域に限定する
//if (coffeeRegion.circularity > 0.50) {
// searchRect = coffeeRegion.predictNextSearchRect ();
// searchRect = ARUtil.calcRectWithinMat (searchRect, cameraMat);
//} else {
// searchRect = new OpenCVForUnity.Rect (0, 0, cameraMat.cols (), cameraMat.rows ());
//}
searchRect = new OpenCVForUnity.Rect(0, 0, cameraMat.cols(), cameraMat.rows());
scorer = new CoffeeScorer(searchRect, coffeeRegion.rect);
v_threshold = 50.0;
}
else if (cup.isTracked())
{
print("Cup is tracked.");
cup.update();
if (cup.culminationAltitude() < 0.5)
{
goto show;
}
// カップの上面の矩形を探索対象矩形とする.
searchRect = cup.getTopSurfaceRect(cameraMat);
scorer = new CoffeeWithCupScorer(searchRect);
v_threshold = 80.0;
}
try {
// 入力画像から探索領域のROI画像を作成
Mat roiMat = new Mat(cameraMat, searchRect);
// HSVチャンネル作成. Vチャンネルで二値化するため
var hsvChannels = ARUtil.getHSVChannels(roiMat);
// 上で決めたVチャンネルの閾値で二値化することによって、輪郭探索の準備をする
// コーヒー表面候補領域と二値画像を作成. すでにROIが絞られているので, 閾値は緩め.
Mat binaryROIMat = new Mat(roiMat.size(), CvType.CV_8UC1);
Imgproc.threshold(hsvChannels [2], binaryROIMat, v_threshold, 255, Imgproc.THRESH_BINARY_INV);
// Imgproc.adaptiveThreshold (hsvChannels [2], coffeeBinaryMat, 255, Imgproc.ADAPTIVE_THRESH_MEAN_C, Imgproc.THRESH_BINARY_INV, 5, 5);
// 膨張させて輪郭抽出を微修正
Imgproc.dilate(hsvChannels [2], hsvChannels [2], Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(5, 5)));
// 二値画像&探索領域矩形で輪郭探索
var contours = ARUtil.findContours(binaryROIMat, searchRect.tl());
// コーヒー領域候補群を作成 -> 円形度で除外 -> 候補にスコアをつける -> 最大スコアの候補を取得
var regionSet = new RegionCandidateSet(contours)
.elliminateByArea(searchRect, 0.01, 0.9)
.elliminateByCircularity(0.2)
.score(scorer)
.sort();
var count = 0;
var regions = new List <Region>();
foreach (var candidate in regionSet.candidates)
{
if (count > 5)
{
break;
}
if (candidate == null)
{
print("candite is null");
break;
}
// 領域作成
coffeeRegion = new Region(candidate, cameraMat);
regions.Add(coffeeRegion);
count++;
}
// テクスチャ作成
texture = textureManager.create(regions);
Mat alphaMask = Mat.zeros(cameraMat.size(), CvType.CV_8U);
foreach (var region in regions)
{
Core.add(region.mask, alphaMask, alphaMask);
}
if (willChange)
{
// アルファブレンド
ARUtil.alphaBlend(cameraMat, texture, alpha, alphaMask);
}
//Debug.Log (candidate.circularity);
// 矩形描画
//Imgproc.rectangle (cameraMat, coffeeRegion.rect.tl (), coffeeRegion.rect.br (), new Scalar (0, 255, 0), 3);
} catch (System.Exception e) {
print(e);
goto show;
}
show:
camQuad1.setMat(cameraMat);
}
void OnPostRender()
{
// UnityのTexture2DからOpencvのMatに変換
UnityEngine.Rect wholeRect = new UnityEngine.Rect(0, 0, cameraTexture.width, cameraTexture.height);
cameraTexture.ReadPixels(wholeRect, 0, 0, true);
Utils.texture2DToMat(cameraTexture, cameraMat);
/* 初期化 */
OpenCVForUnity.Rect searchRect = null; // 探索領域矩形
double v_threshold = 0.0; // 輪郭探索するための二値画像を作るときの明るさの閾値
// Vuforiaでの皿の検出可否によって、以下を切り替える
// ・探索対象領域
if (targetOnPlate)
{
if (plate.isNotFound())
{
// print ("plate is untracked.");
searchRect = new OpenCVForUnity.Rect(0, 0, cameraMat.cols(), cameraMat.rows());
}
else if (plate.isTracked())
{
// カップの上面の矩形を探索対象矩形とする.
searchRect = plate.getTopSurfaceRect(cameraMat);
}
}
else
{
searchRect = new OpenCVForUnity.Rect(0, 0, cameraMat.cols(), cameraMat.rows());
}
if (plate.isTracked())
{
// print ("plate is tracked.");
plate.update();
}
scorer = new MaguroScorer(searchRect);
try
{
var hSVChannels = ARUtil.getHSVChannels(cameraMat);
var yCrCbChannels = ARUtil.getYCrCbChannels(cameraMat);
var matForDeveloper = cameraMat.clone();
// カメラ入力画像から, searchRectサイズの二値画像を生成
Mat binaryROIMat = binaryMatCreator.createBinaryMat(cameraMat, searchRect);
// 二値画像&探索領域矩形で輪郭探索
var contours = ARUtil.findContours(binaryROIMat, searchRect.tl());
// 領域候補群を作成 -> 候補にスコアをつける -> 最大スコアの候補を取得
var regionSet = new RegionCandidateSet(contours)
.elliminateByArea(searchRect, 0.01, 0.9)
.score(scorer)
.sort();
if (targetOnPlate)
{
regionSet = regionSet.elliminateByInclusion(searchRect);
}
if (regionSet.candidates.Count == 0)
{
print("first candidates is 0");
goto show;
}
regionSet.drawRects(matForDeveloper);
var regions = new List <Region>();
foreach (var candidate in regionSet.candidates)
{
// 領域作成
var region = new Region(candidate, cameraMat);
var refinedRegion = region.createRefienedRegion(yCrCbChannels[1], cr_refined_threshold);
if (refinedRegion != null)
{
regions.Add(refinedRegion);
}
}
var filteredRegions = Region.elliminateByInclusionRect(regions);
// 食品領域に貼りつけるテクスチャ作成
var texture = _textureCreator.create(cameraMat, filteredRegions);
if (texture == null)
{
print("regions is empty");
goto show;
}
Mat alphaMask = Mat.zeros(cameraMat.size(), CvType.CV_8U);
foreach (var region in filteredRegions)
{
Core.add(region.mask, alphaMask, alphaMask);
}
if (willChange)
{
_textureCreator.alphaBlend(cameraMat, texture, alphaMask, alpha);
}
// foodRegion.drawRect(matForDeveloper);
Imgproc.rectangle(matForDeveloper, searchRect.tl(), searchRect.br(), new Scalar(0, 0, 255), 3);
//camQuad4.setMat(texture);
//camQuad3.setMat(alphaMask);
camQuad2.setMat(matForDeveloper);
}
catch (System.Exception e)
{
print(e);
goto show;
}
show:
camQuad1.setMat(cameraMat);
}
