private void ConfigureRealSense()
{
try
{
// Create the SenseManager instance
sm = PXCMSenseManager.CreateInstance();
// Enable the color stream
sm.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, ImageWidth, ImageHeight, 30);
// Enable person tracking
sm.EnablePersonTracking();
personModule = sm.QueryPersonTracking();
PXCMPersonTrackingConfiguration personConfig = personModule.QueryConfiguration();
personConfig.SetTrackedAngles(PXCMPersonTrackingConfiguration.TrackingAngles.TRACKING_ANGLES_ALL);
// Enable skeleton tracking - not supported on r200?
//PXCMPersonTrackingConfiguration.SkeletonJointsConfiguration skeletonConfig = personConfig.QuerySkeletonJoints();
//skeletonConfig.Enable();
// Enable the face module
sm.EnableFace();
PXCMFaceModule faceModule = sm.QueryFace();
PXCMFaceConfiguration faceConfig = faceModule.CreateActiveConfiguration();
faceConfig.SetTrackingMode(PXCMFaceConfiguration.TrackingModeType.FACE_MODE_COLOR_PLUS_DEPTH);
faceConfig.strategy = PXCMFaceConfiguration.TrackingStrategyType.STRATEGY_APPEARANCE_TIME;
faceConfig.detection.maxTrackedFaces = 1;
faceConfig.ApplyChanges();
sm.EnableBlob();
PXCMBlobModule blobModule = sm.QueryBlob();
PXCMBlobConfiguration blobConfig = blobModule.CreateActiveConfiguration();
blobConfig.SetMaxBlobs(4); // 4 is the max
blobConfig.SetMaxDistance(2000); // in mm's
blobConfig.ApplyChanges();
//initialize the SenseManager
sm.Init();
faceData = faceModule.CreateOutput();
blobData = blobModule.CreateOutput();
// Mirror the image
sm.QueryCaptureManager().QueryDevice().SetMirrorMode(PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL);
// Release resources
personConfig.Dispose();
faceConfig.Dispose();
faceModule.Dispose();
blobConfig.Dispose();
blobModule.Dispose();
}
catch (Exception)
{
MessageBox.Show("Unable to configure the RealSense camera. Please make sure a R200 camera is connected.", "System Error");
throw;
}
}
private void InitializeBlob()
{
// Blobを取得する
blobModule = senseManager.QueryBlob();
blobData = blobModule.CreateOutput();
var blobConfig = blobModule.CreateActiveConfiguration();
blobConfig.SetContourSmoothing(1.0f);
blobConfig.SetSegmentationSmoothing(1.0f);
blobConfig.SetMaxBlobs(4);
blobConfig.SetMaxDistance(500.0f);
blobConfig.EnableContourExtraction(true);
blobConfig.EnableSegmentationImage(true);
blobConfig.ApplyChanges();
}
private void Uninitialize()
{
if (senseManager != null)
{
senseManager.Dispose();
senseManager = null;
}
if (blobModule != null)
{
blobModule.Dispose();
blobModule = null;
}
if (blobData != null)
{
blobData.Dispose();
blobData = null;
}
}
public MainWindow()
{
InitializeComponent();
// Configure RealSense session and SenseManager interface
session = PXCMSession.CreateInstance();
senseManager = session.CreateSenseManager();
senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 30);
senseManager.EnableBlob();
senseManager.Init();
//Create blobModule from SenseManager
blobModule = senseManager.QueryBlob();
blobConfig = blobModule.CreateActiveConfiguration();
blobConfig.SetMaxBlobs(2);
blobConfig.SetMaxDistance(trackingDistance);
blobConfig.SetMaxObjectDepth(100);
blobConfig.SetMinPixelCount(400);
blobConfig.EnableColorMapping(true);
blobConfig.ApplyChanges();
blobData = blobModule.CreateOutput();
//Audio
WaveOut waveOut;
sineWaveProvider.SetWaveFormat(16000, 1);
sineWaveProvider.Frequency = 1000;
sineWaveProvider.Amplitude = 0;
waveOut = new WaveOut();
waveOut.Init(sineWaveProvider);
waveOut.Play();
startTime = DateTime.Now;
// Start Update thread
update = new Thread(new ThreadStart(Update));
update.Start();
}
void OnDisable()
{
//Disposses all modules
Initialized = false;
if (SenseManager == null)
{
return;
}
DisposeFunctions.ForEach(i => i.DynamicInvoke());
if (FaceModuleOutput != null)
{
FaceModuleOutput.Dispose();
FaceModuleOutput = null;
}
if (HandDataOutput != null)
{
SenseManager.PauseHand(true);
HandDataOutput.Dispose();
HandDataOutput = null;
}
if (BlobDataOutput != null)
{
SenseManager.PauseBlob(true);
BlobDataOutput.Dispose();
BlobDataOutput = null;
}
if (ImageRgbOutput != null)
{
ImageRgbOutput.Dispose();
ImageRgbOutput = null;
}
if (ImageDepthOutput != null)
{
ImageDepthOutput.Dispose();
ImageDepthOutput = null;
}
if (ImageIROutput != null)
{
ImageIROutput.Dispose();
ImageIROutput = null;
}
if (Image3DSegmentationOutput != null)
{
Image3DSegmentationOutput.Dispose();
Image3DSegmentationOutput = null;
}
if (Projection != null)
{
Projection.Dispose();
Projection = null;
}
/* GZ
* if (BlobExtractor != null)
* {
* BlobExtractor.Dispose();
* BlobExtractor = null;
* } */
UvMap = null;
PointCloud = null;
SenseManager.Dispose();
SenseManager = null;
}
void OnEnable()
{
Initialized = false;
/* Create a SenseManager instance */
SenseManager = PXCMSenseManager.CreateInstance();
if (SenseManager == null)
{
print("Unable to create the pipeline instance");
return;
}
if (_speechCommandsRef.Count != 0)
{
SetSenseOption(SenseOption.SenseOptionID.Speech);
}
int numberOfEnabledModalities = 0;
//Set mode according to RunMode - play from file / record / live stream
if (RunMode == MCTTypes.RunModes.PlayFromFile)
{
//CHECK IF FILE EXISTS
if (!System.IO.File.Exists(FilePath))
{
Debug.LogWarning("No Filepath Set Or File Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, false);
Debug.Log("SenseToolkitManager: Playing from file: " + FilePath);
}
}
if (RunMode == MCTTypes.RunModes.RecordToFile)
{
//CHECK IF PATH
string PathOnly = FilePath;
while (!PathOnly[PathOnly.Length - 1].Equals('\\'))
{
PathOnly = PathOnly.Remove(PathOnly.Length - 1, 1);
}
if (!System.IO.Directory.Exists(PathOnly))
{
Debug.LogWarning("No Filepath Set Or Path Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, true);
Debug.Log("SenseToolkitManager: Recording to file: " + FilePath);
}
}
/* Enable modalities according to the set options*/
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
SenseManager.EnableFace();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled = true;
SetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
_sts = SenseManager.EnableHand();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true))
{
_sts = SenseManager.EnableBlob();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
_sts = SenseManager.EnableTracker();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
if (!SpeechManager.IsInitialized)
{
if (SpeechManager.InitalizeSpeech())
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
else
{
UnsetSenseOption(SenseOption.SenseOptionID.Speech);
}
}
else
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoDepthStream, true) ||
IsSenseOptionSet(SenseOption.SenseOptionID.PointCloud, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 0, 0, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoIRStream, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_IR, 0, 0, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoColorStream, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoSegmentation, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
SenseManager.Enable3DSeg();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Enabled = true;
numberOfEnabledModalities++;
}
/* Initialize the execution */
_sts = SenseManager.Init();
if (_sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
//Set different configurations:
// Face
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
var faceModule = SenseManager.QueryFace();
if (faceModule == null)
{
throw new UnityException("QueryFace returned null");
}
var faceConfiguration = faceModule.CreateActiveConfiguration();
if (faceConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
faceConfiguration.Update();
faceConfiguration.detection.isEnabled = true;
faceConfiguration.detection.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.landmarks.isEnabled = true;
faceConfiguration.landmarks.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.pose.isEnabled = true;
faceConfiguration.pose.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.DisableAllAlerts();
faceConfiguration.strategy = PXCMFaceConfiguration.TrackingStrategyType.STRATEGY_APPEARANCE_TIME;
if ((NumberOfDetectedFaces < 1) || (NumberOfDetectedFaces > 15))
{
Debug.Log("Ilegal value for Number Of Detected Faces, value is set to 1");
NumberOfDetectedFaces = 1;
}
faceConfiguration.detection.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.landmarks.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.pose.maxTrackedFaces = NumberOfDetectedFaces;
PXCMFaceConfiguration.ExpressionsConfiguration expressionConfig = faceConfiguration.QueryExpressions();
if (expressionConfig == null)
{
throw new UnityException("QueryExpressions returned null");
}
expressionConfig.Enable();
expressionConfig.EnableAllExpressions();
faceConfiguration.ApplyChanges();
faceConfiguration.Dispose();
FaceModuleOutput = faceModule.CreateOutput();
UnsetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
}
// Hand
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
PXCMHandModule handAnalysis = SenseManager.QueryHand();
if (handAnalysis == null)
{
throw new UnityException("QueryHand returned null");
}
PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration();
if (handConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
handConfiguration.Update();
handConfiguration.EnableAllGestures();
handConfiguration.EnableStabilizer(true);
handConfiguration.DisableAllAlerts();
handConfiguration.EnableSegmentationImage(false);
handConfiguration.ApplyChanges();
handConfiguration.Dispose();
HandDataOutput = handAnalysis.CreateOutput();
}
// Blob
if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true))
{
PXCMBlobModule blobAnalysis = SenseManager.QueryBlob();
if (blobAnalysis == null)
{
throw new UnityException("QueryBlob returned null");
}
PXCMBlobConfiguration blobConfiguration = blobAnalysis.CreateActiveConfiguration();
if (blobConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
blobConfiguration.Update();
blobConfiguration.EnableContourExtraction(true);
blobConfiguration.EnableSegmentationImage(true);
blobConfiguration.EnableStabilizer(true);
blobConfiguration.SetMaxDistance(50 * 10);
blobConfiguration.ApplyChanges();
blobConfiguration.Dispose();
BlobDataOutput = blobAnalysis.CreateOutput();
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled != true)
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
OnDisable();
OnEnable();
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
UpdateSpeechCommands();
SpeechManager.Start();
}
// Create an instance for the projection & blob extractor
if (Projection == null)
{
Projection = SenseManager.QueryCaptureManager().QueryDevice().CreateProjection();
}
/* GZ
* if (BlobExtractor == null)
* {
* SenseManager.session.CreateImpl<PXCMBlobExtractor>(out BlobExtractor);
* }*/
// Set initialization flag
Initialized = true;
}
public void DisplayBlobDataPoints(PXCMBlobData.IBlob bData,int index)
{
if (bitmap == null) return;
lock (this)
{
int sz = 10;
int szPoint = 5;
Graphics g = Graphics.FromImage(bitmap);
using (Pen red = new Pen(Color.Red, 3.0f),
green = new Pen(Color.Green, 3.0f),
darkRed = new Pen(Color.DarkRed, 3.0f))
{
using (SolidBrush brush = new SolidBrush(Color.Green))
{
PXCMPoint3DF32 point =
bData.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_BOTTOM_MOST);
g.DrawEllipse(green, point.x - szPoint, point.y - szPoint, sz, sz);
g.FillEllipse(brush, point.x - szPoint, point.y - szPoint, sz, sz);
point = bData.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_TOP_MOST);
g.DrawEllipse(green, point.x - szPoint, point.y - szPoint, sz, sz);
g.FillEllipse(brush, point.x - szPoint, point.y - szPoint, sz, sz);
point = bData.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_LEFT_MOST);
g.DrawEllipse(green, point.x - szPoint, point.y - szPoint, sz, sz);
g.FillEllipse(brush, point.x - szPoint, point.y - szPoint, sz, sz);
point = bData.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_RIGHT_MOST);
g.DrawEllipse(green, point.x - szPoint, point.y - szPoint, sz, sz);
g.FillEllipse(brush, point.x - szPoint, point.y - szPoint, sz, sz);
point = bData.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER);
brush.Color = Color.DarkRed;
g.DrawEllipse(darkRed, point.x - szPoint, point.y - szPoint, sz, sz);
g.FillEllipse(brush, point.x - szPoint, point.y - szPoint, sz, sz);
point = bData.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CLOSEST);
brush.Color = Color.Red;
g.DrawEllipse(red, point.x - szPoint/2, point.y - szPoint/2, szPoint, szPoint);
g.FillEllipse(brush, point.x - szPoint, point.y - szPoint, sz, sz);
}
}
g.Dispose();
}
}
/* Displaying Mask Images */
private unsafe void DisplayPicture(PXCMImage depth, PXCMBlobData blobData)
{
if (depth == null)
return;
PXCMImage image = depth;
PXCMImage.ImageInfo info = image.QueryInfo();
int numOfBlobs = blobData.QueryNumberOfBlobs();
if (_maxBlobToShow > numOfBlobs)
{
_maxBlobToShow = numOfBlobs;
}
PXCMBlobData.IBlob[] blobList = new PXCMBlobData.IBlob[_maxBlobToShow];
PXCMPointI32[][] pointOuter = new PXCMPointI32[_maxBlobToShow][];
PXCMPointI32[][] pointInner = new PXCMPointI32[_maxBlobToShow][];
Bitmap picture = new Bitmap(image.info.width, image.info.height, PixelFormat.Format32bppRgb);
PXCMImage.ImageData bdata;
pxcmStatus results = pxcmStatus.PXCM_STATUS_NO_ERROR;
PXCMBlobData.AccessOrderType accessOrder = PXCMBlobData.AccessOrderType.ACCESS_ORDER_LARGE_TO_SMALL;
int accessOrderBy = form.GetAccessOrder();
switch (accessOrderBy)
{
case 1:
accessOrder = PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR;
break;
case 2:
accessOrder = PXCMBlobData.AccessOrderType.ACCESS_ORDER_RIGHT_TO_LEFT;
break;
case 0:
default:
accessOrder = PXCMBlobData.AccessOrderType.ACCESS_ORDER_LARGE_TO_SMALL;
break;
}
Rectangle rect = new Rectangle(0, 0, image.info.width, image.info.height);
BitmapData bitmapdata = picture.LockBits(rect, ImageLockMode.ReadWrite, picture.PixelFormat);
for (int j = 0; j < _maxBlobToShow; j++)
{
byte tmp1 = (Byte)(255 - (255 / (_maxBlobToShow) * j));
results = blobData.QueryBlobByAccessOrder(j, accessOrder, out blobList[j]);
if (results == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
bool isSegmentationImage = true;
results = blobList[j].QuerySegmentationImage(out image);
if (results != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMImage.ImageInfo imgInfo = new PXCMImage.ImageInfo();
imgInfo.width = 640;
imgInfo.height = 480;
imgInfo.format = PXCMImage.PixelFormat.PIXEL_FORMAT_Y8;
PXCMSession session = PXCMSession.CreateInstance();
if (session != null)
{
image = session.CreateImage(info);
if (image == null) return;
}
image.AcquireAccess(PXCMImage.Access.ACCESS_WRITE, PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out bdata);
byte* numPtr = (byte*)bitmapdata.Scan0; //dst
byte* numPtr2 = (byte*)bdata.planes[0]; //row
int imagesize = image.info.width * image.info.height;
byte tmp;
for (int i = 0; i < imagesize; i++, numPtr += 4, numPtr2++)
{
tmp = (byte)(0);
numPtr[0] = tmp;
numPtr[1] = tmp;
numPtr[2] = tmp;
numPtr[3] = tmp;
}
image.ReleaseAccess(bdata);
isSegmentationImage = false;
}
results = image.AcquireAccess(PXCMImage.Access.ACCESS_READ_WRITE, PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out bdata);
if (form.GetBlobState() && isSegmentationImage == true)
{
byte* numPtr = (byte*)bitmapdata.Scan0; //dst
byte* numPtr2 = (byte*)bdata.planes[0]; //row
int imagesize = image.info.width * image.info.height;
for (int i = 0; i < imagesize; i++, numPtr += 4, numPtr2++)
{
byte tmp = (Byte)(numPtr2[0] == 0 ? 0 : tmp1);
tmp |= numPtr[0];
numPtr[0] = tmp;
numPtr[1] = tmp;
numPtr[2] = tmp;
numPtr[3] = 0xff;
}
}
if ((form.GetContourState()))
{
int contourNumber = blobList[j].QueryNumberOfContours();
if (contourNumber > 0)
{
for (int k = 0; k < contourNumber; ++k)
{
int contourSize = blobList[j].QueryContourSize(k);
if (blobList[j].IsContourOuter(k) == true)
blobList[j].QueryContourPoints(k, out pointOuter[j]);
else
{
blobList[j].QueryContourPoints(k, out pointInner[j]);
}
}
if (results == pxcmStatus.PXCM_STATUS_NO_ERROR && form.GetBlobState() == false)
{
byte* numPtr = (byte*)bitmapdata.Scan0; //dst
byte* numPtr2 = (byte*)bdata.planes[0]; //row
int imagesize = image.info.width * image.info.height;
byte tmp;
for (int i = 0; i < imagesize; i++, numPtr += 4, numPtr2++)
{
tmp = (byte)(0);
numPtr[0] = tmp;
numPtr[1] = tmp;
numPtr[2] = tmp;
numPtr[3] = tmp;
}
}
}
}
image.ReleaseAccess(bdata);
image.Dispose();
}
}
picture.UnlockBits(bitmapdata);
form.DisplayBitmap(picture);
///////// that is my polygon zone
Bitmap imageInstance2 = picture;
i++;
Bitmap croppedImage = null;
if (contourMostRight.x > 0) {
int rectWidth = (int)(contourMostRight.x - contourMostLeft.x);
int rectHeight = (int)(contourMostTop.y - contourMostBottom.y);
Rectangle sourceRectangle = new Rectangle(new Point((int)contourMostLeft.x,
(int)contourMostBottom.y),
new Size(rectWidth, rectHeight));
croppedImage = CropImage(imageInstance2, sourceRectangle);
}
String[] origArray = {
"d:\\origG.jpeg",
"d:\\origX.jpeg",
"d:\\origY.jpeg"
};
for (int i = 0; i < origArray.Length; i++)
{
Bitmap orig = null;
if (File.Exists(origArray[i]))
orig = new Bitmap(@origArray[i]);
if (orig != null && croppedImage != null)
{
float diff = 0;
orig = ScaleImage(orig, 150, 100);
croppedImage = ScaleImage(croppedImage, 150, 100);
bool isImageBlank = true;
for (int y = 0; y < orig.Height; y++)
{
for (int x = 0; x < orig.Width; x++)
{
if(croppedImage.GetPixel(x, y).R > 1 && croppedImage.GetPixel(x, y).B > 1
&& croppedImage.GetPixel(x, y).G > 1)
{
isImageBlank = false;
break;
}
}
}
if (!isImageBlank && orig.Size.Width == croppedImage.Size.Width)
{
for (int y = 0; y < orig.Height; y++)
{
for (int x = 0; x < orig.Width; x++)
{
diff += (float)Math.Abs(orig.GetPixel(x, y).R -
croppedImage.GetPixel(x, y).R) / 255;
diff += (float)Math.Abs(orig.GetPixel(x, y).G -
croppedImage.GetPixel(x, y).G) / 255;
diff += (float)Math.Abs(orig.GetPixel(x, y).B -
croppedImage.GetPixel(x, y).B) / 255;
}
}
}
float percentMatch = 100 * diff / (orig.Width * orig.Height * 2);
Console.WriteLine("diff: {0} %", percentMatch);
if (percentMatch >= 90){
Console.WriteLine(origArray[i].ToString());
Environment.Exit(0);
}
}
}
/*
if (croppedImage != null)
{
croppedImage.Save("d:\\cropedImage.jpeg", System.Drawing.Imaging.ImageFormat.Jpeg);
// croppedImage.Save("d:\\origG.jpeg", System.Drawing.Imaging.ImageFormat.Jpeg);
}*/
if (i == 100)
Environment.Exit(0);
picture.Dispose();
for (int i = 0; i < _maxBlobToShow; i++)
{
if (form.GetContourState())
{
if (pointOuter[i] != null && pointOuter[i].Length > 0)
form.DisplayContour(pointOuter[i], i);
if (pointInner[i] != null && pointInner[i].Length > 0)
form.DisplayContour(pointInner[i], i);
}
if (form.GetBlobDataPointsState())
{
form.DisplayBlobDataPoints(blobList[i], i + 1);
}
PXCMPoint3DF32 point = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER);
contourMostRight = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_LEFT_MOST);
contourMostLeft = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_RIGHT_MOST);
contourMostBottom = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_TOP_MOST);
contourMostTop = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_BOTTOM_MOST);
form.DisplayBlobNumber(point, i + 1);
}
}
/// <summary>
/// Using PXCMSenseManager to handle data
/// </summary>
public void SimplePipeline()
{
session = PXCMSession.CreateInstance();
if (session != null)
{
instance = session.CreateSenseManager();
if (instance == null)
{
Debug.Log("Create SenseManager Failure");
return;
}
pxcmStatus status = instance.EnableBlob();
PXCMBlobModule blobModule = instance.QueryBlob();
if (status != pxcmStatus.PXCM_STATUS_NO_ERROR || blobModule == null)
{
Debug.Log("Failed Loading Module");
return;
}
blobConfiguration = blobModule.CreateActiveConfiguration();
blobData = blobModule.CreateOutput();
if (blobConfiguration != null)
{
blobConfiguration.SetSegmentationSmoothing(1.0f);
blobConfiguration.SetMaxDistance(550);
blobConfiguration.SetMaxObjectDepth(100);
blobConfiguration.SetMaxBlobs(_maxBlobToShow);
blobConfiguration.SetContourSmoothing(1.0f);
blobConfiguration.EnableContourExtraction(true);
blobConfiguration.EnableSegmentationImage(true);
blobConfiguration.ApplyChanges();
}
if (blobData == null)
{
Debug.Log("Failed Create Output");
return;
}
PXCMSenseManager.Handler handler = new PXCMSenseManager.Handler();
handler.onModuleProcessedFrame = new PXCMSenseManager.Handler.OnModuleProcessedFrameDelegate(OnNewFrame);
if (instance.Init(handler) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMCapture.DeviceInfo dinfo;
instance.QueryCaptureManager().QueryDevice().QueryDeviceInfo(out dinfo);
if (dinfo != null && dinfo.model == PXCMCapture.DeviceModel.DEVICE_MODEL_IVCAM)
{
instance.QueryCaptureManager().QueryDevice().SetMirrorMode(PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED);
}
/* Set the depth stream confidence threshold value - Any depth pixels with a confidence score below the threshold will be set to the low confidence pixel value*/
instance.QueryCaptureManager().QueryDevice().SetDepthConfidenceThreshold(10);
/* Set the smoothing aggressiveness parameter - High smoothing effect for distances between 850mm to 1000mm bringing good accuracy with moderate sharpness level.*/
instance.QueryCaptureManager().QueryDevice().SetIVCAMFilterOption(6);
}
}
else
{
Debug.Log("Init Failed");
}
}
public void Dispose()
{
if(handData != null)
{
handData.Dispose();
handData = null;
}
if(blobData != null)
{
blobData.Dispose();
blobData = null;
}
if(this.device != null)
{
this.device.Dispose();
}
if(this.cm != null)
{
this.cm.Dispose();
}
if(this.sm != null)
{
this.sm.Close();
this.sm.Dispose();
}
this.Session.Dispose();
}
private void UpdateContoursImage( PXCMBlobData.IBlob blob, int index )
{
// 輪郭を表示する
var numOfContours = blob.QueryNumberOfContours();
for ( int i = 0; i < numOfContours; ++i ) {
// 輪郭の点の数を取得する
var size = blob.QueryContourSize( i );
if ( size <= 0 ) {
continue;
}
// 輪郭の点を取得する
PXCMPointI32[] points;
var sts = blob.QueryContourPoints( i, out points );
if ( sts < pxcmStatus.PXCM_STATUS_NO_ERROR ) {
continue;
}
// 輪郭の点を描画する
drawContour( points, index );
}
}
private void Uninitialize()
{
if ( senseManager != null ) {
senseManager.Dispose();
senseManager = null;
}
if ( blobModule != null ) {
blobModule.Dispose();
blobModule = null;
}
if ( blobData != null ) {
blobData.Dispose();
blobData = null;
}
}
private void InitializeBlob()
{
// Blobを取得する
blobModule = senseManager.QueryBlob();
blobData = blobModule.CreateOutput();
var blobConfig = blobModule.CreateActiveConfiguration();
blobConfig.SetContourSmoothing( 1.0f );
blobConfig.SetSegmentationSmoothing( 1.0f );
blobConfig.SetMaxBlobs( 4 );
blobConfig.SetMaxDistance( 500.0f );
blobConfig.EnableContourExtraction( true );
blobConfig.EnableSegmentationImage( true );
blobConfig.ApplyChanges();
}
void OnDisable()
{
//Disposses all modules
Initialized = false;
if (SenseManager == null) return;
DisposeFunctions.ForEach( i=> i.DynamicInvoke());
if (FaceModuleOutput != null)
{
FaceModuleOutput.Dispose();
FaceModuleOutput = null;
}
if (HandDataOutput != null)
{
SenseManager.PauseHand(true);
HandDataOutput.Dispose();
HandDataOutput = null;
}
if (BlobDataOutput != null)
{
SenseManager.PauseBlob(true);
BlobDataOutput.Dispose();
BlobDataOutput = null;
}
if (ImageRgbOutput != null)
{
ImageRgbOutput.Dispose();
ImageRgbOutput = null;
}
if (ImageDepthOutput != null)
{
ImageDepthOutput.Dispose();
ImageDepthOutput = null;
}
if (ImageIROutput != null)
{
ImageIROutput.Dispose();
ImageIROutput = null;
}
if (Image3DSegmentationOutput != null)
{
Image3DSegmentationOutput.Dispose();
Image3DSegmentationOutput = null;
}
if (Projection != null)
{
Projection.Dispose();
Projection = null;
}
/* GZ
if (BlobExtractor != null)
{
BlobExtractor.Dispose();
BlobExtractor = null;
} */
UvMap = null;
PointCloud = null;
SenseManager.Dispose();
SenseManager = null;
}
void OnEnable()
{
Initialized = false;
/* Create a SenseManager instance */
SenseManager = PXCMSenseManager.CreateInstance();
if (SenseManager == null)
{
print("Unable to create the pipeline instance");
return;
}
if (_speechCommandsRef.Count != 0)
{
SetSenseOption(SenseOption.SenseOptionID.Speech);
}
int numberOfEnabledModalities = 0;
//Set mode according to RunMode - play from file / record / live stream
if (RunMode == MCTTypes.RunModes.PlayFromFile)
{
//CHECK IF FILE EXISTS
if (!System.IO.File.Exists(FilePath))
{
Debug.LogWarning("No Filepath Set Or File Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, false);
Debug.Log("SenseToolkitManager: Playing from file: " + FilePath);
}
}
if (RunMode == MCTTypes.RunModes.RecordToFile)
{
//CHECK IF PATH
string PathOnly = FilePath;
while (!PathOnly[PathOnly.Length - 1].Equals('\\'))
{
PathOnly = PathOnly.Remove(PathOnly.Length - 1, 1);
}
if (!System.IO.Directory.Exists(PathOnly))
{
Debug.LogWarning("No Filepath Set Or Path Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, true);
Debug.Log("SenseToolkitManager: Recording to file: " + FilePath);
}
}
/* Enable modalities according to the set options*/
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
SenseManager.EnableFace();
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Face).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Face).Enabled = true;
SetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
_sts = SenseManager.EnableHand();
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Hand).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Hand).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true))
{
_sts = SenseManager.EnableBlob();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
_sts = SenseManager.EnableTracker();
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Object).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
if (!SpeechManager.IsInitialized)
{
if (SpeechManager.InitalizeSpeech())
{
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
else
{
UnsetSenseOption(SenseOption.SenseOptionID.Speech);
}
}
else
{
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoDepthStream, true) ||
IsSenseOptionSet(SenseOption.SenseOptionID.PointCloud, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 0, 0, 0);
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoIRStream, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_IR, 0, 0, 0);
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoColorStream, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoSegmentation, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
SenseManager.Enable3DSeg();
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Enabled = true;
numberOfEnabledModalities++;
}
/* Initialize the execution */
_sts = SenseManager.Init();
if (_sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (numberOfEnabledModalities > 0)
{
Debug.LogError("Unable to initialize all modalities");
}
return;
}
//Set different configurations:
// Face
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
var faceModule = SenseManager.QueryFace();
var faceConfiguration = faceModule.CreateActiveConfiguration();
if (faceConfiguration == null) throw new UnityException("CreateActiveConfiguration returned null");
faceConfiguration.Update();
faceConfiguration.detection.isEnabled = true;
faceConfiguration.detection.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.landmarks.isEnabled = true;
faceConfiguration.landmarks.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.pose.isEnabled = true;
faceConfiguration.pose.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.DisableAllAlerts();
faceConfiguration.strategy = PXCMFaceConfiguration.TrackingStrategyType.STRATEGY_APPEARANCE_TIME;
if ((NumberOfDetectedFaces < 1) || (NumberOfDetectedFaces > 15))
{
Debug.Log("Ilegal value for Number Of Detected Faces, value is set to 1");
NumberOfDetectedFaces = 1;
}
faceConfiguration.detection.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.landmarks.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.pose.maxTrackedFaces = NumberOfDetectedFaces;
PXCMFaceConfiguration.ExpressionsConfiguration expressionConfig = faceConfiguration.QueryExpressions();
expressionConfig.Enable();
expressionConfig.EnableAllExpressions();
faceConfiguration.ApplyChanges();
faceConfiguration.Dispose();
FaceModuleOutput = faceModule.CreateOutput();
UnsetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
}
// Hand
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
PXCMHandModule handAnalysis = SenseManager.QueryHand();
PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration();
if (handConfiguration == null) throw new UnityException("CreateActiveConfiguration returned null");
handConfiguration.Update();
handConfiguration.EnableAllGestures();
handConfiguration.EnableStabilizer(true);
handConfiguration.DisableAllAlerts();
handConfiguration.EnableSegmentationImage(false);
handConfiguration.ApplyChanges();
handConfiguration.Dispose();
HandDataOutput = handAnalysis.CreateOutput();
}
// Blob
if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true))
{
PXCMBlobModule blobAnalysis = SenseManager.QueryBlob();
PXCMBlobConfiguration blobConfiguration = blobAnalysis.CreateActiveConfiguration();
if (blobConfiguration == null) throw new UnityException("CreateActiveConfiguration returned null");
blobConfiguration.Update();
blobConfiguration.EnableContourExtraction(true);
blobConfiguration.EnableSegmentationImage(true);
blobConfiguration.EnableStabilizer(true);
blobConfiguration.SetMaxDistance(50 * 10);
blobConfiguration.ApplyChanges();
blobConfiguration.Dispose();
BlobDataOutput = blobAnalysis.CreateOutput();
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
if (_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Object).Enabled != true)
{
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
OnDisable();
OnEnable();
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
UpdateSpeechCommands();
SpeechManager.Start();
}
// Create an instance for the projection & blob extractor
if (Projection == null)
{
Projection = SenseManager.QueryCaptureManager().QueryDevice().CreateProjection();
}
/* GZ
if (BlobExtractor == null)
{
SenseManager.session.CreateImpl<PXCMBlobExtractor>(out BlobExtractor);
}*/
// Set initialization flag
Initialized = true;
}
