protected override void UpdateDisplayList(float width, float height)
{
//if (Id == "hbox")
// Debug.Log("UpdateDisplayList: " + width + "; " + height);
//if (Id == "contentGroup")
// Debug.Log("GroupBase.UpdateDisplayList: " + width + ", " + height);
base.UpdateDisplayList(width, height);
if (_layoutInvalidateDisplayListFlag)
{
_layoutInvalidateDisplayListFlag = false;
if (_autoLayout && null != _layout)
{
_layout.UpdateDisplayList(width, height);
}
if (null != _layout)
{
_layout.UpdateScrollRect(width, height);
}
}
if (_drawingListInvalidated)
{
//Debug.Log("Calling DepthUtil.UpdateDrawingList on " + this);
_drawingListInvalidated = false;
//DepthUtil.UpdateDrawingList(this);
DepthUtil.UpdateDrawingList(_contentPane ?? this);
}
}
/// <summary>
/// Finds the contours with minimum perimeters.
/// </summary>
/// <param name="image"></param>
/// <param name="hand">Can be null.</param>
private void FindContours(Joint hand)
{
// Non-zero pixels are treated as 1s. Source image content is modifield.
CvInvoke.cvFindContours(HandMask.Ptr, storage, ref contourPtr, StructSize.MCvContour,
RETR_TYPE.CV_RETR_EXTERNAL, CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, new Point(0, 0));
var contour = new Seq <Point>(contourPtr, null);
float z = DefaultZDist;
if (hand != null)
{
z = hand.Position.Z;
}
HandCandidates.Clear();
double perimThresh = DepthUtil.GetDepthImageLength(width, HandWidth, z) * 2;
for (; contour != null && contour.Ptr.ToInt32() != 0; contour = contour.HNext)
{
var perim = CvInvoke.cvContourPerimeter(contour.Ptr);
if (perim > perimThresh)
{
HandMask.Draw(contour, new Gray(255), -1);
HandCandidates.Add(contour.BoundingRectangle);
}
}
}
protected override void Subscribed()
{
_sprite.TilePosition = new Vector3(
_npc.Waypoints[0].X,
_npc.Waypoints[0].Y,
DepthUtil.OutdoorCharacterDepth(_npc.Waypoints[0].Y));
}
/// <summary>
/// </summary>
/// <returns></returns>
Rectangle ComputeInitialRect(float z)
{
if (InterestPoints.Count > 0)
{
int x = 0, y = 0;
foreach (Point ip in InterestPoints)
{
x += ip.X;
y += ip.Y;
}
x /= InterestPoints.Count;
y /= InterestPoints.Count;
double xx = 0, yy = 0;
foreach (Point ip in InterestPoints)
{
xx += (ip.X - x) * (ip.X - x);
yy += (ip.Y - y) * (ip.Y - y);
}
xx /= InterestPoints.Count;
yy /= InterestPoints.Count;
xx = Math.Sqrt(xx);
yy = Math.Sqrt(yy);
var scaledHandWidth = DepthUtil.GetDepthImageLength(width, HandWidth, z);
var rectWidth = (int)Math.Max(xx, scaledHandWidth);
var rectHeight = (int)Math.Max(yy, scaledHandWidth);
return(new Rectangle(x - rectWidth / 2, y - rectHeight / 2, rectWidth, rectHeight));
}
return(HandRect);
}
protected override void CommitProperties()
{
base.CommitProperties();
// Handle a deferred state change request.
if (_currentStateDeferred != null)
{
var newState = _currentStateDeferred;
_currentStateDeferred = null;
CurrentState = newState;
}
// Typically state changes occur immediately, but during
// component initialization we defer until commitProperties to
// reduce a bit of the startup noise.
if (_currentStateChanged && !Initialized)
{
_currentStateChanged = false;
CommitCurrentState();
}
if (_drawingListInvalidated)
{
//Debug.Log("Calling DepthUtil.UpdateDrawingList on " + this);
_drawingListInvalidated = false;
DepthUtil.UpdateDrawingList(this);
}
}
void depthDisplay_MouseDown(object sender, MouseButtonEventArgs e)
{
var p = e.GetPosition(depthDisplay);
var x = (int)(p.X * HandInputParams.DepthWidth / depthDisplay.ActualWidth);
var y = (int)(p.Y * HandInputParams.DepthHeight / depthDisplay.ActualHeight);
var raw = depthManager.PixelData[y * HandInputParams.DepthWidth + x];
Log.DebugFormat("depth = {0}", DepthUtil.RawToDepth(raw));
}
///// <summary>
///// The list of components ordered by child Depth<br/>
///// Used for rendering<br/>
///// The rendering order might have different than layout order
///// </summary>
//public virtual List<DisplayListMember> DrawingList
//{
// get
// {
// return _drawingList;
// }
//}
/// <summary>
/// Updates the depth list which is used for rendering<br/>
/// The depth list is based upon the Depth property of each child<br/>
/// It is the same as the order list if no Depth is defined on each of the children
/// </summary>
internal virtual void InvalidateDrawingList()
{
// display object updates the list immediatelly (there are no invalidation methods)
//if (this is Stage)
// Debug.Log(string.Format(" DOC: Updating drawing list -> {0} [{1}, {2}]", this, _children.Count, _drawingList.Count));
if (AutoUpdateDrawingList)
{
DepthUtil.UpdateDrawingList(this);
}
}
void ConvertDepthImage(short[] depthFrame)
{
var data = Gray.Data;
var scale = (float)255 / HandInputParams.MaxDepth;
for (int r = 0; r < height; r++)
{
for (int c = 0; c < width; c++)
{
var index = r * width + c;
var pixel = depthFrame[index];
var depth = DepthUtil.RawToDepth(pixel);
depth = (depth <HandInputParams.MinDepth || depth> HandInputParams.MaxDepth) ?
HandInputParams.MaxDepth : depth;
data[r, c, 0] = (byte)((HandInputParams.MaxDepth - depth) * scale);
}
}
CvInvoke.cvResize(Gray.Ptr, smallGray.Ptr, INTER.CV_INTER_LINEAR);
}
/// <summary>
/// Computes initial hand searching rectangle in the depth image. If the point is out side of
/// the frame boundary, an empty rectangle will be returned.
/// </summary>
/// <returns></returns>
Rectangle ComputeInitialRect(DepthImagePoint point, float z)
{
var scaledHandWidth = DepthUtil.GetDepthImageLength(width, HandWidth, z) * 2;
var left = Math.Max(0, (int)(point.X - scaledHandWidth / 2));
left = Math.Min(left, width);
var top = Math.Max(0, (int)(point.Y - scaledHandWidth / 2));
top = Math.Min(top, height);
// right and bottom are exclusive.
var right = Math.Min(width, (int)(left + scaledHandWidth));
var bottom = Math.Min(height, (int)(top + scaledHandWidth));
var rectWidth = right - left;
var rectHeight = bottom - top;
if (rectWidth <= 0 || rectHeight <= 0)
{
return(Rectangle.Empty);
}
return(new Rectangle(left, top, right - left, bottom - top));
}
private Image <Gray, Byte> CreatePlayerImage(short[] depthFrame)
{
CvInvoke.cvZero(playerMask.Ptr);
var data = playerMask.Data;
for (int r = 0; r < height; r++)
{
for (int c = 0; c < width; c++)
{
var index = r * width + c;
short pixel = depthFrame[index];
int playerIndex = DepthUtil.RawToPlayerIndex(pixel);
if (playerIndex > 0)
{
data[r, c, 0] = 255;
}
}
}
return(playerMask);
}
/// <exception cref="Db4objects.Db4o.Ext.Db4oIOException"></exception>
private void CollectIDsByInstantiatingCollection(QueryingReadContext context)
{
int id = context.CollectionID();
if (id == 0)
{
return;
}
Transaction transaction = context.Transaction();
ObjectContainerBase container = context.Container();
object obj = container.GetByID(transaction, id);
if (obj == null)
{
return;
}
// FIXME: [TA] review activation depth
int depth = DepthUtil.AdjustDepthToBorders(2);
container.Activate(transaction, obj, container.ActivationDepthProvider().ActivationDepth
(depth, ActivationMode.Activate));
Platform4.ForEachCollectionElement(obj, new _IVisitor4_390(context));
}
private void AlignColorImage(short[] depthFrame, Image <Gray, Byte> colorImage)
{
CvInvoke.cvZero(alignedImg.Ptr);
var data = colorImage.Data;
var alignedData = alignedImg.Data;
for (int r = 0; r < height; r++)
{
for (int c = 0; c < width; c++)
{
var depthPixel = depthFrame[r * width + c];
var depth = DepthUtil.RawToDepth(depthPixel);
var cp = mapper.MapDepthPointToColorPoint(c, r, depth);
if (cp.X >= 0 && cp.X < width && cp.Y >= 0 && cp.Y < height)
{
alignedData[r, c, 0] = data[cp.Y, cp.X, 0];
}
}
}
CvInvoke.cvMorphologyEx(alignedImg.Ptr, alignedImg.Ptr, IntPtr.Zero, IntPtr.Zero,
CV_MORPH_OP.CV_MOP_CLOSE, 1);
}
/// <summary>
/// TODO - do the Depth thing for each child
/// </summary>
/// <param name="child"></param>
internal virtual void BringChildToFront(DisplayListMember child)
{
DepthUtil.BringToFront(_drawingList, child);
}
/// <summary>
/// TODO - do the Depth thing for each child
/// </summary>
/// <param name="child"></param>
internal virtual void PushChildToBack(DisplayListMember child)
{
DepthUtil.PushToBack(_drawingList, child);
}
/// <summary>
/// If no bounding box is found, returns the last bounding box.
/// </summary>
/// <returns></returns>
List <Rectangle> FindBestBoundingBox(short[] depthFrame, Skeleton skeleton)
{
CvInvoke.cvConvert(SaliencyProb.Ptr, TempMask.Ptr);
// Non-zero pixels are treated as 1s. Source image content is modifield.
CvInvoke.cvFindContours(TempMask.Ptr, storage, ref contourPtr, StructSize.MCvContour,
RETR_TYPE.CV_RETR_EXTERNAL, CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, new Point(0, 0));
var contour = new Seq <Point>(contourPtr, null);
SortedList <float, Seq <Point> > bestContours = new SortedList <float, Seq <Point> >();
List <Rectangle> bestBoundingBoxes = new List <Rectangle>();
float z = DefaultZDist;
var hand = SkeletonUtil.GetJoint(skeleton, JointType.HandRight);
if (hand != null)
{
z = hand.Position.Z;
}
double perimThresh = DepthUtil.GetDepthImageLength(width, HandWidth, z) * 2;
FaceModel = SkeletonUtil.GetFaceModel(skeleton, mapper);
for (; contour != null && contour.Ptr.ToInt32() != 0; contour = contour.HNext)
{
var perim = CvInvoke.cvContourPerimeter(contour.Ptr);
if (perim > perimThresh)
{
var rect = contour.BoundingRectangle;
var score = ContourScore(rect);
var center = rect.Center();
int x = (int)center.X;
int y = (int)center.Y;
var depth = DepthUtil.RawToDepth(depthFrame[y * width + x]);
if (!FaceModel.IsPartOfFace(x, y, depth) &&
(bestContours.Count < NumTrackedHands || score > bestContours.ElementAt(0).Key))
{
bestContours.Add(score, contour);
if (bestContours.Count > NumTrackedHands)
{
bestContours.RemoveAt(0);
}
}
}
}
if (bestContours.Count > 0)
{
foreach (var c in bestContours.Values)
{
var rect = c.BoundingRectangle;
CvInvoke.cvCamShift(TrackedDepthFrame.Ptr, rect, new MCvTermCriteria(CamShiftIter),
out connectedComp, out shiftedBox);
var bestBoundingBox = shiftedBox.MinAreaRect();
bestBoundingBoxes.Add(bestBoundingBox);
//FloodFill(TrackedDepthFrame, bestBoundingBox);
//if (bestBoundingBox.Width > 0) {
// TempMask.ROI = bestBoundingBox;
// CvInvoke.cvFindContours(TempMask.Ptr, storage, ref contourPtr, StructSize.MCvContour,
// RETR_TYPE.CV_RETR_EXTERNAL, CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE,
// new Point(0, 0));
// contour = new Seq<Point>(contourPtr, null);
// Seq<Point> largestContour = null;
// var maxPerim = perimThresh;
// for (; contour != null && contour.Ptr.ToInt32() != 0; contour = contour.HNext) {
// var perim = CvInvoke.cvContourPerimeter(contour.Ptr);
// if (perim > maxPerim) {
// maxPerim = perim;
// largestContour = contour;
// }
// }
// CvInvoke.cvZero(TempMask.Ptr);
// if (largestContour != null)
// TempMask.Draw(largestContour, new Gray(255), -1);
// FilterImage(TrackedDepthFrame, TempMask);
// TempMask.ROI = Rectangle.Empty;
//}
}
}
return(bestBoundingBoxes);
}
// TODO code duplication in fixed activation/update depth
public virtual FixedActivationDepth AdjustDepthToBorders
()
{
return(new FixedActivationDepth(DepthUtil.AdjustDepthToBorders
(_depth)));
}
private SkeletonPoint FindHand(short[] depthData, Rectangle rect)
{
CvInvoke.cvZero(HandImage.Ptr);
var handImageData = HandImage.Data;
var handMaskData = HandMask.Data;
var playerMaskData = playerMask.Data;
var maxDepth = 0;
var minDepth = Int32.MaxValue;
for (int y = rect.Top; y < rect.Top + rect.Height && y < height; y++)
{
for (int x = rect.Left; x < rect.Left + rect.Width && x < width; x++)
{
if (y > 0 && x > 0 && handMaskData[y, x, 0] > 0 &&
playerMaskData[y, x, 0] > 0)
{
var depth = DepthUtil.RawToDepth(depthData[y * width + x]);
maxDepth = Math.Max(maxDepth, depth);
if (depth < minDepth)
{
minDepth = depth;
}
}
}
}
var scale = (float)255 / (maxDepth - minDepth);
for (int y = rect.Top; y < rect.Top + rect.Height && y < height; y++)
{
for (int x = rect.Left; x < rect.Left + rect.Width && x < width; x++)
{
if (y > 0 && x > 0 && playerMaskData[y, x, 0] > 0 &&
handMaskData[y, x, 0] > 0)
{
var depth = DepthUtil.RawToDepth(depthData[y * width + x]);
handImageData[y, x, 0] = (byte)((maxDepth - depth) * scale);
}
}
}
var connectedComp = new MCvConnectedComp();
var shiftedBox = new MCvBox2D();
CvInvoke.cvCamShift(HandImage.Ptr, rect, new MCvTermCriteria(0.0), out connectedComp,
out shiftedBox);
PrevHand = new windows.Point(HandBox.center.X, HandBox.center.Y);
HandBox = shiftedBox;
var newRect = shiftedBox.MinAreaRect();
var aveDepth = 0.0;
var count = 0;
for (int y = newRect.Top; y < newRect.Top + newRect.Height && y < height; y++)
{
for (int x = newRect.Left; x < newRect.Left + newRect.Width && x < width; x++)
{
if (x > 0 && y > 0 && playerMaskData[y, x, 0] > 0 &&
handMaskData[y, x, 0] > 0)
{
var depth = DepthUtil.RawToDepth(depthData[y * width + x]);
aveDepth += depth;
count++;
}
}
}
aveDepth /= count;
var shiftedCenterX = Math.Max(0, shiftedBox.center.X);
shiftedCenterX = Math.Min(shiftedCenterX, width);
var shiftedCenterY = Math.Max(0, shiftedBox.center.Y);
shiftedCenterY = Math.Min(shiftedCenterY, height);
return(mapper.MapDepthPointToSkeletonPoint((int)shiftedCenterX,
(int)shiftedCenterY, (int)aveDepth));
}
private void CreateContentPane()
{
if (null != _contentPane)
{
return;
}
//Debug.Log("Creating content pane");
CreatingContentPane = true;
var n = NumberOfChildren; // snapshot now
//Debug.Log("CreateContentPane. Number of children: " + n);
System.Collections.Generic.List <DisplayListMember> childrenToMove = new System.Collections.Generic.List <DisplayListMember>();
for (int i = 0; i < n; i++)
{
childrenToMove.Add(base.GetChildAt(i));
//Debug.Log("Will move: " + base.GetChildAt(i));
}
/**
* Content pane is a simple display object
* NOTE: we have to use temp variable here
* The reason is this line below: newPane.AddChild(child);
* If the _contentPane is not null, this changed the flow:
* we are expecting that AddChild() indirectly calls the RemoveChild() on parent (meaning: this container)
* However, if _contentPane alsready set, it will try to remove the child from the pane itself!
* */
#if DEBUG
var newPane = new DisplayObjectContainer
{
Id = "content_pane", // for debugging purposes
X = 0,
Y = 0,
AutoUpdateDrawingList = false,
Visible = true
};
#endif
#if !DEBUG
var newPane = new DisplayObjectContainer
{
X = 0,
Y = 0,
AutoUpdateDrawingList = false,
Visible = true
};
#endif
/**
* Add content pane as a last child
* (cannot use AddChild(_contentPane) here, because it takes the number of
* children internally depending of the pane existance)
* Important:
* Also cannot use AddChildAt, since it would then try to add content pane to the content pane itself
* */
base.AddingChild(newPane);
QAddChildAt(newPane, n);
base.ChildAdded(newPane);
//var mover = new ChildMover(this, _contentPane, numberOfChildren);
//mover.Move();
foreach (DisplayListMember child in childrenToMove)
{
// set the container as a parent
var cmp = child as Component;
//RemoveChild(child); // TODO: remove
newPane.AddChild(child); // AddChild interno zove RemoveChild na OVOM kontejneru. Zbog toga je potrebno da je _contentPane == null
if (null != cmp)
{
cmp.ParentChanged(newPane);
}
//Debug.Log(" ... done");
}
_contentPane = newPane;
//Debug.Log("NumberOfChildren: " + NumberOfChildren);
//Debug.Log("_contentPane.NumberOfChildren: " + _contentPane.NumberOfChildren);
DepthUtil.UpdateDrawingList(this); // important! Cannot call the InvalidateDrawingList here because it will never update the display list of this
//DepthUtil.UpdateDrawingList(_contentPane); // auto update is turned off on the content pane
InvalidateDrawingList(); // same as DepthUtil.UpdateDrawingList(_contentPane) but delayed (so perhaps better for performance)
CreatingContentPane = false;
_contentPane.Visible = true;
}
// TODO code duplication in fixed activation/update depth
public virtual FixedUpdateDepth AdjustDepthToBorders
()
{
return(ForDepth(DepthUtil.AdjustDepthToBorders(_depth)));
}