public RealsenseManager(bool isDataOnly)
{
sm = RS.SenseManager.CreateInstance();
if (!isDataOnly)
{
rgbReader = RS.SampleReader.Activate(sm);
depthReader = RS.SampleReader.Activate(sm);
rgbReader.EnableStream(RS.StreamType.STREAM_TYPE_COLOR);
depthReader.EnableStream(RS.StreamType.STREAM_TYPE_DEPTH);
rgbReader.SampleArrived += RgbReader_SampleArrived;
depthReader.SampleArrived += DepthReader_SampleArrived;
}
// config hand
handModule = HandModule.Activate(sm);
handConfig = handModule.CreateActiveConfiguration();
handConfig.TrackedJointsEnabled = true;
handConfig.StabilizerEnabled = true;
handConfig.TrackingMode = TrackingModeType.TRACKING_MODE_FULL_HAND;
handConfig.ApplyChanges();
handData = handModule.CreateOutput();
handModule.FrameProcessed += HandModule_FrameProcessed;
}
public void EnableGesturesFromSelection()
{
ActivatedGestures = form.GetSelectedGestures();
if (HandConfiguration != null)
{
if (ActivatedGestures.Count != 0)
{
HandConfiguration.DisableAllGestures();
foreach (string gestureName in ActivatedGestures)
{
if (HandConfiguration.IsGestureEnabled(gestureName) == false)
{
HandConfiguration.EnableGesture(gestureName, true);
HandConfiguration.ApplyChanges(); // Ev. aus Schleife nehmen
}
}
}
// No gestures activated
else
{
HandConfiguration.DisableAllGestures();
HandConfiguration.ApplyChanges();
}
}
}
public void RegisterHandEvents()
{
if (HandConfiguration != null)
{
HandConfiguration.AlertFired += OnFiredAlert;
HandConfiguration.GestureFired += OnFiredGesture;
HandConfiguration.ApplyChanges();
}
}
// Setup Hand Configuration
void SetupHandConfiguration()
{
handConfiguration = handModule.CreateActiveConfiguration();
if (handConfiguration == null)
{
throw new System.Exception("Failed Creating Hand Configuration");
}
else
{
Debug.Log(TAG + "Successful Creating Hand Configuration");
CreateHandData();
}
}
public void CleanUpHands()
{
// Clean Up
if (handData != null)
{
handData.Dispose();
}
if (HandConfiguration != null)
{
HandConfiguration.Dispose();
}
foreach (Image Image in _mImages)
{
Image.Dispose();
}
}
public void SetUpHandModule()
{
/* Set Module */
handModule = HandModule.Activate(manager.SenseManager);
if (handModule == null)
{
manager.SetStatus("Failed Loading Module");
manager.Stop = true;
return;
}
HandConfiguration = handModule.CreateActiveConfiguration();
if (HandConfiguration == null)
{
manager.SetStatus("Failed Create Configuration");
manager.Stop = true;
return;
}
handData = handModule.CreateOutput();
if (handData == null)
{
manager.SetStatus("Failed Create Output");
HandConfiguration.Dispose();
manager.Stop = true;
return;
}
HandConfiguration.TrackingMode = TrackingModeType.TRACKING_MODE_FULL_HAND;
HandConfiguration.TrackedJointsEnabled = true;
HandConfiguration.EnableJointSpeed(JointType.JOINT_INDEX_TIP, JointSpeedType.JOINT_SPEED_ABSOLUTE, 20);
HandConfiguration.StabilizerEnabled = true;
HandConfiguration.EnableAllAlerts();
HandConfiguration.SegmentationImageEnabled = false;
HandConfiguration.SmoothingValue = 1; // The value is from 0(not smoothed) to 1(smoothed motion).
HandConfiguration.ApplyChanges();
}
/* Using SenseManager to handle data */
public void SimplePipeline()
{
_form.UpdateInfo(String.Empty, System.Drawing.Color.Black);
bool liveCamera = false;
bool flag = true;
SenseManager instance = null;
_disconnected = false;
instance = _form.session.CreateSenseManager();
if (instance == null)
{
_form.UpdateStatus("Failed creating SenseManager");
_form.EnableTrackingMode(true);
return;
}
CaptureManager captureManager = instance.CaptureManager;
DeviceInfo info = null;
if (captureManager != null)
{
if (_form.GetPlaybackState())
{
captureManager.SetFileName(_form.GetFileName(), false);
info = _form.GetDeviceFromFileMenu(_form.GetFileName());
}
else
{
if (_form.Devices.Count == 0)
{
_form.UpdateStatus("No device were found");
return;
}
_form.Devices.TryGetValue(_form.GetCheckedDevice(), out info);
if (_form.GetRecordState())
{
captureManager.SetFileName(_form.GetFileName(), true);
if (_form.Devices.TryGetValue(_form.GetCheckedDevice(), out info))
{
captureManager.FilterByDeviceInfo(_form.GetCheckedDeviceInfo());
}
}
else
{
captureManager.FilterByDeviceInfo(_form.GetCheckedDeviceInfo());
liveCamera = true;
}
if (info == null)
{
_form.UpdateStatus("Device Failure");
return;
}
}
}
/* Set Module */
HandModule handAnalysis;
// SenseManager.Handler handler = new SenseManager Handler();
// handler.onModuleProcessedFrame = new SenseManager.Handler.OnModuleProcessedFrameDelegate(OnNewFrame);
HandConfiguration handConfiguration = null;
HandData handData = null;
handAnalysis = HandModule.Activate(instance);
if (handAnalysis == null)
{
_form.UpdateStatus("Failed Loading Module");
_form.EnableTrackingMode(true);
return;
}
handConfiguration = handAnalysis.CreateActiveConfiguration();
if (handConfiguration == null)
{
_form.UpdateStatus("Failed Create Configuration");
_form.EnableTrackingMode(true);
instance.Close();
instance.Dispose();
return;
}
handData = handAnalysis.CreateOutput();
if (handData == null)
{
_form.UpdateStatus("Failed Create Output");
_form.EnableTrackingMode(true);
handConfiguration.Dispose();
instance.Close();
instance.Dispose();
return;
}
FPSTimer timer = new FPSTimer(_form);
_form.UpdateStatus("Init Started");
if (instance.Init() == Status.STATUS_NO_ERROR)
{
DeviceInfo dinfo;
DeviceModel dModel = DeviceModel.DEVICE_MODEL_F200;
Device device = instance.CaptureManager.Device;
if (device != null)
{
device.QueryDeviceInfo(out dinfo);
dModel = dinfo.model;
_maxRange = device.DepthSensorRange.max;
}
if (handConfiguration != null)
{
TrackingModeType trackingMode = TrackingModeType.TRACKING_MODE_FULL_HAND;
if (_form.GetFullHandModeState())
{
trackingMode = TrackingModeType.TRACKING_MODE_FULL_HAND;
}
handConfiguration.TrackingMode = trackingMode;
handConfiguration.EnableAllAlerts();
handConfiguration.SegmentationImageEnabled = true;
bool isEnabled = handConfiguration.SegmentationImageEnabled;
handConfiguration.ApplyChanges();
_form.resetGesturesList();
int totalNumOfGestures = handConfiguration.NumberOfGestures;
if (totalNumOfGestures > 0)
{
this._form.UpdateGesturesToList("", 0);
for (int i = 0; i < totalNumOfGestures; i++)
{
string gestureName = string.Empty;
if (handConfiguration.QueryGestureNameByIndex(i, out gestureName) ==
Status.STATUS_NO_ERROR)
{
this._form.UpdateGesturesToList(gestureName, i + 1);
}
}
_form.UpdateGesturesListSize();
}
}
_form.UpdateStatus("Streaming");
int frameCounter = 0;
int frameNumber = 0;
while (!_form.stop)
{
string gestureName = _form.GetGestureName();
if (handConfiguration != null)
{
if (string.IsNullOrEmpty(gestureName) == false)
{
if (handConfiguration.IsGestureEnabled(gestureName) == false)
{
handConfiguration.DisableAllGestures();
handConfiguration.EnableGesture(gestureName, true);
handConfiguration.ApplyChanges();
}
}
else
{
handConfiguration.DisableAllGestures();
handConfiguration.ApplyChanges();
}
}
if (instance.AcquireFrame(true) < Status.STATUS_NO_ERROR)
{
break;
}
frameCounter++;
if (!DisplayDeviceConnection(!instance.IsConnected()))
{
Sample sample = instance.Sample;
if (sample != null && sample.Depth != null)
{
frameNumber = liveCamera ? frameCounter : instance.CaptureManager.FrameIndex;
if (handData != null)
{
handData.Update();
DisplayPicture(sample.Depth, handData);
DisplayGesture(handData, frameNumber);
DisplayJoints(handData);
DisplayAlerts(handData, frameNumber);
}
_form.UpdatePanel();
}
timer.Tick();
}
instance.ReleaseFrame();
}
}
else
{
_form.UpdateStatus("Init Failed");
flag = false;
}
foreach (Image Image in _mImages)
{
Image.Dispose();
}
// Clean Up
if (handData != null)
{
handData.Dispose();
}
if (handConfiguration != null)
{
handConfiguration.Dispose();
}
instance.Close();
instance.Dispose();
if (flag)
{
_form.UpdateStatus("Stopped");
}
}
private void computeMeanValuesAndHandConfiguration(List <SGInstance> instances)
{
Vec3 LeftPalmPosTotal = new Vec3();
Vec3 LeftPalmSphereCenterTotal = new Vec3();
float LeftPalmSphereRadiusTotal = 0;
float LeftYawTotal = 0;
float LeftPitchTotal = 0;
float LeftRollTotal = 0;
var LeftFingersExtendedTotal = new Dictionary <Finger.FingerType, int>();
var LeftFingerTipPositionsTotal = new Dictionary <Finger.FingerType, Vec3>();
Vec3 RightPalmPosTotal = new Vec3();
Vec3 RightPalmSphereCenterTotal = new Vec3();
float RightPalmSphereRadiusTotal = 0;
float RightYawTotal = 0;
float RightPitchTotal = 0;
float RightRollTotal = 0;
var RightFingersExtendedTotal = new Dictionary <Finger.FingerType, int>();
var RightFingerTipPositionsTotal = new Dictionary <Finger.FingerType, Vec3>();
// Initialize FingersExtended Dicts
foreach (var fingerType in (Finger.FingerType[])Enum.GetValues(typeof(Finger.FingerType)))
{
LeftFingersExtendedTotal.Add(fingerType, 0);
RightFingersExtendedTotal.Add(fingerType, 0);
LeftFingerTipPositionsTotal.Add(fingerType, new Vec3());
RightFingerTipPositionsTotal.Add(fingerType, new Vec3());
}
// Keep track of how many instances of each hand config.
// If 31 instances of right hand only and 14 instances of left only,
// consider the class right hand only.
var handConfigCounts = new Dictionary <HandConfiguration, int>()
{
{ HandConfiguration.NoHands, 0 },
{ HandConfiguration.LeftHandOnly, 0 },
{ HandConfiguration.RightHandOnly, 0 },
{ HandConfiguration.BothHands, 0 },
};
// Keep track of hands' positions relative to each other (left to right)
Vec3 LeftToRightHandTotal = new Vec3();
foreach (var instance in instances)
{
var leftPalmPos = new Vec3(); // for getting hands positions relative to each other totals
var rightPalmPos = new Vec3();
handConfigCounts[instance.HandConfiguration]++;
foreach (var hand in instance.Hands)
{
if (hand.IsLeft)
{
leftPalmPos = hand.PalmPosition;
LeftPalmPosTotal += hand.PalmPosition;
LeftPalmSphereCenterTotal += hand.PalmSphereCenter;
LeftPalmSphereRadiusTotal += hand.PalmSphereRadius;
LeftYawTotal += hand.Yaw;
LeftPitchTotal += hand.Pitch;
LeftRollTotal += hand.Roll;
foreach (var fingerType in (Finger.FingerType[])Enum.GetValues(typeof(Finger.FingerType)))
{
if (hand.FingersExtended[fingerType])
{
LeftFingersExtendedTotal[fingerType]++;
}
LeftFingerTipPositionsTotal[fingerType] += hand.FingerJointPositions_Relative[fingerType][Finger.FingerJoint.JOINT_TIP];
}
}
else
{
rightPalmPos = hand.PalmPosition;
RightPalmPosTotal += hand.PalmPosition;
RightPalmSphereCenterTotal += hand.PalmSphereCenter;
RightPalmSphereRadiusTotal += hand.PalmSphereRadius;
RightYawTotal += hand.Yaw;
RightPitchTotal += hand.Pitch;
RightRollTotal += hand.Roll;
foreach (var fingerType in (Finger.FingerType[])Enum.GetValues(typeof(Finger.FingerType)))
{
if (hand.FingersExtended[fingerType])
{
RightFingersExtendedTotal[fingerType]++;
}
RightFingerTipPositionsTotal[fingerType] += hand.FingerJointPositions_Relative[fingerType][Finger.FingerJoint.JOINT_TIP];
}
}
}
LeftToRightHandTotal += leftPalmPos - rightPalmPos;
}
HandConfiguration = handConfigCounts.FirstOrDefault(h => h.Value == handConfigCounts.Values.Max()).Key;
MeanValues = new Dictionary <FeatureName, object>();
float numInstances = instances.Count; // Needs to be float for divisions
MeanValues.Add(FeatureName.LeftPalmPosition, LeftPalmPosTotal / numInstances);
MeanValues.Add(FeatureName.LeftPalmSphereCenter, LeftPalmSphereCenterTotal / numInstances);
MeanValues.Add(FeatureName.LeftPalmSphereRadius, LeftPalmSphereRadiusTotal / numInstances);
MeanValues.Add(FeatureName.LeftYaw, LeftYawTotal / numInstances);
MeanValues.Add(FeatureName.LeftPitch, LeftPitchTotal / numInstances);
MeanValues.Add(FeatureName.LeftRoll, LeftRollTotal / numInstances);
MeanValues.Add(FeatureName.RightPalmPosition, RightPalmPosTotal / numInstances);
MeanValues.Add(FeatureName.RightPalmSphereCenter, RightPalmSphereCenterTotal / numInstances);
MeanValues.Add(FeatureName.RightPalmSphereRadius, RightPalmSphereRadiusTotal / numInstances);
MeanValues.Add(FeatureName.RightYaw, RightYawTotal / numInstances);
MeanValues.Add(FeatureName.RightPitch, RightPitchTotal / numInstances);
MeanValues.Add(FeatureName.RightRoll, RightRollTotal / numInstances);
var averagedLeftFingerTipPositions = new Dictionary <Finger.FingerType, Vec3>();
var averagedRightFingerTipPositions = new Dictionary <Finger.FingerType, Vec3>();
var leftFingersExtendedPercentages = new Dictionary <Finger.FingerType, float>();
var rightFingersExtendedPercentages = new Dictionary <Finger.FingerType, float>();
foreach (var fingerType in (Finger.FingerType[])Enum.GetValues(typeof(Finger.FingerType)))
{
averagedLeftFingerTipPositions.Add(fingerType, LeftFingerTipPositionsTotal[fingerType] / numInstances);
averagedRightFingerTipPositions.Add(fingerType, RightFingerTipPositionsTotal[fingerType] / numInstances);
// Set MeanValue for FingersExtended as a float: The percentage that are true.
leftFingersExtendedPercentages.Add(fingerType, LeftFingersExtendedTotal[fingerType] / numInstances);
rightFingersExtendedPercentages.Add(fingerType, RightFingersExtendedTotal[fingerType] / numInstances);
}
MeanValues.Add(FeatureName.LeftFingerTipPositions, averagedLeftFingerTipPositions);
MeanValues.Add(FeatureName.RightFingerTipPositions, averagedRightFingerTipPositions);
MeanValues.Add(FeatureName.LeftFingersExtended, leftFingersExtendedPercentages);
MeanValues.Add(FeatureName.RightFingersExtended, rightFingersExtendedPercentages);
MeanValues.Add(FeatureName.LeftToRightHand, LeftToRightHandTotal / numInstances);
}
public void SetHandConfiguration(HandConfiguration hand_config)
{
_hand_config = hand_config;
}
