private static AbstractDrawing CreateDrawing(Metadata metadata, long timestamp, OpenPosePerson person)
{
// We only support files created using the BODY_25 model, not COCO or MPI.
if (person.pose_keypoints_2d != null && person.pose_keypoints_2d.Count != 75)
{
return(null);
}
string toolName = "OpenPoseBody25";
DrawingToolGenericPosture tool = ToolManager.Tools[toolName] as DrawingToolGenericPosture;
if (tool == null)
{
return(null);
}
GenericPosture posture = GenericPostureManager.Instanciate(tool.ToolId, true);
ParsePosture(posture, person);
DrawingGenericPosture drawing = new DrawingGenericPosture(tool.ToolId, PointF.Empty, posture, timestamp, metadata.AverageTimeStampsPerFrame, ToolManager.GetStylePreset(toolName));
drawing.Name = "OpenPose";
// Disable onion skinning.
drawing.InfosFading.UseDefault = false;
drawing.InfosFading.ReferenceTimestamp = timestamp;
drawing.InfosFading.AverageTimeStampsPerFrame = metadata.AverageTimeStampsPerFrame;
drawing.InfosFading.AlwaysVisible = false;
drawing.InfosFading.OpaqueFrames = 1;
drawing.InfosFading.FadingFrames = 0;
return(drawing);
}
public DrawingGenericPosture(Guid toolId, PointF origin, GenericPosture posture, long timestamp, long averageTimeStampsPerFrame, DrawingStyle stylePreset)
{
this.toolId = toolId;
this.origin = origin;
this.genericPosture = posture;
if (genericPosture != null)
{
Init();
}
// Decoration and binding to mini editors.
styleHelper.Bicolor = new Bicolor(Color.Empty);
styleHelper.Font = new Font("Arial", 12, FontStyle.Bold);
if (stylePreset == null)
{
stylePreset = new DrawingStyle();
stylePreset.Elements.Add("line color", new StyleElementColor(Color.DarkOliveGreen));
}
style = stylePreset.Clone();
BindStyle();
// Fading
infosFading = new InfosFading(timestamp, averageTimeStampsPerFrame);
menuFlipHorizontal.Click += menuFlipHorizontal_Click;
menuFlipHorizontal.Image = Properties.Drawings.fliphorizontal;
menuFlipVertical.Click += menuFlipVertical_Click;
menuFlipVertical.Image = Properties.Drawings.flipvertical;
}
public static void MoveHandle(GenericPosture posture, CalibrationHelper calibrationHelper, int handle, PointF point, Keys modifiers)
{
try
{
// Update the point(s) attached to the handle based on the constraints.
// Update all other points that may have been impacted.
switch (posture.Handles[handle].Type)
{
case HandleType.Point:
MovePointHandle(posture, calibrationHelper, handle, point, modifiers);
break;
case HandleType.Segment:
MoveSegmentHandle(posture, calibrationHelper, handle, point);
break;
case HandleType.Ellipse:
case HandleType.Circle:
MoveCircleHandle(posture, handle, point);
break;
}
}
catch (Exception e)
{
log.DebugFormat("Error while moving handle");
log.DebugFormat(e.ToString());
}
}
private static void MovePointHandleByRotationSteps(GenericPosture posture, CalibrationHelper calibrationHelper, int handle, PointF point, GenericPostureConstraintRotationSteps constraint)
{
if (constraint == null)
{
return;
}
PointF parent = posture.PointList[constraint.Origin];
PointF leg1 = posture.PointList[constraint.Leg1];
if (parent == leg1 || constraint.Step == 0)
{
return;
}
PointF candidate = point;
if (constraint.KeepDistance)
{
PointF leg2 = posture.PointList[posture.Handles[handle].Reference];
float distance = GeometryHelper.GetDistance(parent, leg2);
candidate = GeometryHelper.GetPointAtDistance(parent, point, distance);
}
int constraintAngleSubdivisions = 360 / constraint.Step;
posture.PointList[posture.Handles[handle].Reference] = GeometryHelper.GetPointAtClosestRotationStep(parent, leg1, candidate, constraintAngleSubdivisions);
}
private static void AlignPointPerpendicular(GenericPosture posture, CalibrationHelper calibrationHelper, GenericPosturePerpendicularAlign impact)
{
// The point is moved so that it stays on a perpendicular segment relatively to another segment.
if (impact == null)
{
return;
}
PointF pivot = posture.PointList[impact.Origin];
PointF leg1 = posture.PointList[impact.Leg1];
PointF pointToMove = posture.PointList[impact.PointToMove];
if (pivot == leg1)
{
return;
}
PointF pivotPlane = calibrationHelper.GetPoint(pivot);
PointF leg1Plane = calibrationHelper.GetPoint(leg1);
PointF pointPlane = calibrationHelper.GetPoint(pointToMove);
PointF resultPlane = GeometryHelper.GetPointAtAngle(pivotPlane, leg1Plane, pointPlane, 90);
PointF result = calibrationHelper.GetImagePoint(resultPlane);
posture.PointList[impact.PointToMove] = result;
}
private static void ProcessPointImpacts(GenericPosture posture, CalibrationHelper calibrationHelper, int handle, PointF old)
{
foreach (GenericPostureAbstractImpact impact in posture.Handles[handle].Impacts)
{
ProcessPointImpact(posture, calibrationHelper, impact, handle, old);
}
}
private static void MoveSegmentHandleAlongVertical(GenericPosture posture, int handle, PointF point)
{
Vector moved = new Vector(posture.Handles[handle].GrabPoint, point);
Vector v = new Vector(0, moved.Y);
TranslateSegmentHandle(posture, handle, v);
}
private static void MoveCircleHandleFreely(GenericPosture posture, int handle, PointF point)
{
PointF center = posture.PointList[posture.Circles[posture.Handles[handle].Reference].Center];
Vector v = new Vector(center, point);
float radius = v.Norm();
posture.Circles[posture.Handles[handle].Reference].Radius = (int)radius;
}
private static void TranslateSegmentHandle(GenericPosture posture, int handle, Vector vector)
{
int start = posture.Segments[posture.Handles[handle].Reference].Start;
int end = posture.Segments[posture.Handles[handle].Reference].End;
TranslateSegment(posture, start, end, vector);
posture.Handles[handle].GrabPoint = posture.Handles[handle].GrabPoint + vector;
}
private static void MovePointHandle(GenericPosture posture, CalibrationHelper calibrationHelper, int handle, PointF point, Keys modifiers)
{
// Constraints. (position of the point managed by this handle).
GenericPostureAbstractConstraint constraint = posture.Handles[handle].Constraint;
PointF old = posture.PointList[posture.Handles[handle].Reference];
PrepareImpacts(posture, handle);
if (constraint == null ||
(!string.IsNullOrEmpty(constraint.OptionGroup) && !posture.OptionGroups[constraint.OptionGroup]))
{
MovePointHandleFreely(posture, handle, point);
}
else
{
switch (constraint.Type)
{
case ConstraintType.None:
MovePointHandleFreely(posture, handle, point);
break;
case ConstraintType.LineSlide:
MovePointHandleAlongLine(posture, handle, point, constraint as GenericPostureConstraintLineSlide);
break;
case ConstraintType.VerticalSlide:
MovePointHandleAlongVertical(posture, calibrationHelper, handle, point);
break;
case ConstraintType.HorizontalSlide:
MovePointHandleAlongHorizontal(posture, handle, point);
break;
case ConstraintType.DistanceToPoint:
MovePointHandleAtDistance(posture, handle, point, constraint as GenericPostureConstraintDistanceToPoint, modifiers);
break;
case ConstraintType.RotationSteps:
MovePointHandleByRotationSteps(posture, calibrationHelper, handle, point, constraint as GenericPostureConstraintRotationSteps);
break;
case ConstraintType.PerpendicularSlide:
MovePointHandleAlongPerpendicular(posture, calibrationHelper, handle, point, constraint as GenericPostureConstraintPerpendicularSlide);
break;
case ConstraintType.ParallelSlide:
MovePointHandleAlongParallel(posture, calibrationHelper, handle, point, constraint as GenericPostureConstraintParallelSlide);
break;
case ConstraintType.LockedInPlace:
break;
}
}
ProcessPointImpacts(posture, calibrationHelper, handle, old);
}
private static void PivotPoints(GenericPosture posture, float radians, GenericPostureImpactPivot impact)
{
// Rotates a series of point around a pivot point.
PointF pivot = posture.PointList[impact.Pivot];
foreach (int pointRef in impact.Impacted)
{
posture.PointList[pointRef] = GeometryHelper.Pivot(pivot, posture.PointList[pointRef], radians);
}
}
public override AbstractDrawing GetNewDrawing(PointF origin, long timestamp, long averageTimeStampsPerFrame, IImageToViewportTransformer transformer)
{
if (ToolManager.Tools.ContainsKey(name))
{
stylePreset = ToolManager.GetStylePreset(name);
}
GenericPosture posture = GenericPostureManager.Instanciate(id, false);
return(new DrawingGenericPosture(origin, posture, timestamp, averageTimeStampsPerFrame, stylePreset));
}
public PointF ComputeLocation(GenericPosture posture)
{
PointF result = PointF.Empty;
if (reference >= 0 && reference < posture.Points.Count)
{
result = posture.Points[reference];
}
return(result);
}
private static void AlignPointHorizontal(GenericPosture posture, int handle, GenericPostureImpactHorizontalAlign impact)
{
if (impact == null)
{
return;
}
PointF impacted = posture.PointList[impact.PointRef];
PointF impacting = posture.PointList[posture.Handles[handle].Reference];
posture.PointList[impact.PointRef] = new PointF(impacting.X, impacted.Y);
}
private static void MovePointHandleAlongVertical(GenericPosture posture, CalibrationHelper calibrationHelper, int handle, PointF point)
{
/*PointF source = calibrationHelper.GetPoint(posture.Points[posture.Handles[handle].Reference]);
* PointF target = calibrationHelper.GetPoint(point.ToPointF());
*
* PointF result = new PointF(source.X, target.Y);
* PointF resultImage = calibrationHelper.GetImagePoint(result);
*
* posture.Points[posture.Handles[handle].Reference] = resultImage;*/
posture.Points[posture.Handles[handle].Reference] = new PointF(posture.Points[posture.Handles[handle].Reference].X, point.Y);
}
private static void MovePointHandleAlongLine(GenericPosture posture, int handle, PointF point, GenericPostureConstraintLineSlide constraint)
{
if (constraint == null)
{
MovePointHandleFreely(posture, handle, point);
return;
}
PointF start = posture.PointList[constraint.Start];
PointF end = posture.PointList[constraint.End];
posture.PointList[posture.Handles[handle].Reference] = GeometryHelper.GetClosestPoint(start, end, point, constraint.AllowedPosition, constraint.Margin);
}
private static void MoveSegmentSymmetrically(GenericPosture posture, GenericPostureImpactHorizontalSymmetry impact)
{
// Moves a segment so it is symmetric to another segment relatively to a vertical symmetry axis.
GenericPostureSegment impacting = posture.Segments[impact.Impacting];
GenericPostureSegment impacted = posture.Segments[impact.Impacted];
GenericPostureSegment axis = posture.Segments[impact.Axis];
Vector vStart = new Vector(posture.PointList[impacting.Start], posture.PointList[axis.Start]);
posture.PointList[impacted.Start] = new PointF(posture.PointList[axis.Start].X + vStart.X, posture.PointList[impacting.Start].Y);
Vector vEnd = new Vector(posture.PointList[impacting.End], posture.PointList[axis.End]);
posture.PointList[impacted.End] = new PointF(posture.PointList[axis.End].X + vEnd.X, posture.PointList[impacting.End].Y);
}
private static void ProcessPointImpact(GenericPosture posture, CalibrationHelper calibrationHelper, GenericPostureAbstractImpact impact, int handle, PointF old)
{
switch (impact.Type)
{
case ImpactType.LineAlign:
AlignPointSegment(posture, impact as GenericPostureImpactLineAlign);
break;
case ImpactType.VerticalAlign:
AlignPointVertical(posture, calibrationHelper, handle, impact as GenericPostureImpactVerticalAlign);
break;
case ImpactType.HorizontalAlign:
AlignPointHorizontal(posture, handle, impact as GenericPostureImpactHorizontalAlign);
break;
case ImpactType.Pivot:
// Get rotation that was applied. Apply same rotation on all points.
GenericPostureImpactPivot impactPivot = impact as GenericPostureImpactPivot;
if (impact != null)
{
PointF a = posture.PointList[impactPivot.Pivot];
PointF b = old;
PointF c = posture.PointList[posture.Handles[handle].Reference];
float radians = GeometryHelper.GetAngle(a, b, c);
PivotPoints(posture, radians, impact as GenericPostureImpactPivot);
}
break;
case ImpactType.KeepAngle:
KeepPointAngle(posture, impact as GenericPostureImpactKeepAngle);
break;
case ImpactType.SegmentCenter:
SegmentCenter(posture, calibrationHelper, impact as GenericPostureImpactSegmentCenter);
break;
case ImpactType.PerdpendicularAlign:
AlignPointPerpendicular(posture, calibrationHelper, impact as GenericPosturePerpendicularAlign);
break;
case ImpactType.ParallelAlign:
AlignPointParallel(posture, calibrationHelper, impact as GenericPostureParallelAlign);
break;
}
}
private static void AlignPointVertical(GenericPosture posture, CalibrationHelper calibrationHelper, int handle, GenericPostureImpactVerticalAlign impact)
{
if (impact == null)
{
return;
}
/*PointF impacted = calibrationHelper.GetPoint(posture.Points[impact.PointRef]);
* PointF impacting = calibrationHelper.GetPoint(posture.Points[posture.Handles[handle].Reference]);
*
* PointF result = calibrationHelper.GetImagePoint(new PointF(impacted.X, impacting.Y));
* posture.Points[impact.PointRef] = result;*/
PointF impacted = posture.PointList[impact.PointRef];
PointF impacting = posture.PointList[posture.Handles[handle].Reference];
posture.PointList[impact.PointRef] = new PointF(impacted.X, impacting.Y);
}
public void SetInfo(GenericPosture posture)
{
this.id = posture.Id;
if (!string.IsNullOrEmpty(posture.Name))
{
name = posture.Name;
}
if (!string.IsNullOrEmpty(posture.DisplayName))
{
displayName = posture.DisplayName;
}
if (posture.Icon != null && posture.Icon.Width == 16 && posture.Icon.Height == 16)
{
icon = posture.Icon;
}
}
private static void MovePointHandleByRotationSteps(GenericPosture posture, CalibrationHelper calibrationHelper, int handle, PointF point, GenericPostureConstraintRotationSteps constraint)
{
if (constraint == null)
{
return;
}
PointF parent = posture.Points[constraint.Origin];
PointF leg1 = posture.Points[constraint.Leg1];
if (parent == leg1 || constraint.Step == 0)
{
return;
}
int constraintAngleSubdivisions = 360 / constraint.Step;
posture.Points[posture.Handles[handle].Reference] = GeometryHelper.GetPointAtClosestRotationStep(parent, leg1, point, constraintAngleSubdivisions);
}
private static void MoveCircleHandle(GenericPosture posture, int handle, PointF point)
{
// Constraints. (position of the point managed by this handle).
GenericPostureAbstractConstraint constraint = posture.Handles[handle].Constraint;
if (!IsActive(constraint, posture))
{
MoveCircleHandleFreely(posture, handle, point);
}
else
{
switch (constraint.Type)
{
case ConstraintType.None:
MoveCircleHandleFreely(posture, handle, point);
break;
}
}
}
public PointF ComputeLocation(GenericPosture posture)
{
PointF result = PointF.Empty;
if (weightedPoints.Count == 2)
{
// Special case, weight is optionnal if the other point is weighted.
IWeightedPoint p0 = weightedPoints[0];
IWeightedPoint p1 = weightedPoints[1];
float w0 = p0.Weight;
float w1 = p1.Weight;
if (w0 == 0)
{
w0 = 1 - w1;
}
else if (w1 == 0)
{
w1 = 1 - w0;
}
PointF l0 = p0.ComputeLocation(posture);
PointF l1 = p1.ComputeLocation(posture);
float x = (l0.X * w0) + (l1.X * w1);
float y = (l0.Y * w0) + (l1.Y * w1);
result = new PointF(x, y);
}
else
{
foreach (IWeightedPoint weightedPoint in weightedPoints)
{
PointF location = weightedPoint.ComputeLocation(posture);
PointF scaled = location.Scale(weightedPoint.Weight, weightedPoint.Weight);
result = result.Translate(scaled.X, scaled.Y);
}
}
LastPoint = result;
return(result);
}
private static void AlignPointSegment(GenericPosture posture, GenericPostureImpactLineAlign impact)
{
if (impact == null)
{
return;
}
PointF start = posture.PointList[impact.Start];
PointF end = posture.PointList[impact.End];
if (start == end)
{
posture.PointList[impact.PointToAlign] = start;
return;
}
PointF result = GeometryHelper.GetClosestPoint(start, end, posture.PointList[impact.PointToAlign], PointLinePosition.Anywhere, 10);
posture.PointList[impact.PointToAlign] = result;
}
private static void ParsePosture(GenericPosture posture, OpenPosePerson person)
{
// We assume pixel values for x and y. (See flag keypoint_scale)
// The order of entries in the custom tool is the same.
for (int i = 0; i < person.pose_keypoints_2d.Count; i += 3)
{
float x = person.pose_keypoints_2d[i + 0];
float y = person.pose_keypoints_2d[i + 1];
float c = person.pose_keypoints_2d[i + 2];
int index = i / 3;
posture.PointList[index] = new PointF(x, y);
// Visibility of the point and incoming segments depends on confidence.
if (options.ContainsKey(index) && posture.Options.ContainsKey(options[index]))
{
posture.Options[options[index]].Value = c >= confidenceThreshold;
}
}
}
private static void KeepPointAngle(GenericPosture posture, GenericPostureImpactKeepAngle impact)
{
// The point is moved so that the angle between its leg and the other leg is kept.
if (impact == null)
{
return;
}
PointF origin = posture.PointList[impact.Origin];
PointF leg1 = posture.PointList[impact.Leg1];
if (origin == leg1)
{
return;
}
PointF result = GeometryHelper.GetPointAtAngleAndDistance(origin, leg1, impact.OldAngle, impact.OldDistance);
posture.PointList[impact.Leg2] = result;
}
private static void MovePointHandleAlongParallel(GenericPosture posture, CalibrationHelper calibrationHelper, int handle, PointF point, GenericPostureConstraintParallelSlide constraint)
{
if (constraint == null)
{
return;
}
PointF a = posture.PointList[constraint.A];
PointF b = posture.PointList[constraint.B];
PointF c = posture.PointList[constraint.C];
PointF aPlane = calibrationHelper.GetPoint(a);
PointF bPlane = calibrationHelper.GetPoint(b);
PointF cPlane = calibrationHelper.GetPoint(c);
PointF pointPlane = calibrationHelper.GetPoint(point);
PointF resultPlane = GeometryHelper.GetPointOnParallel(aPlane, bPlane, cPlane, pointPlane);
PointF result = calibrationHelper.GetImagePoint(resultPlane);
posture.PointList[posture.Handles[handle].Reference] = result;
}
private static void SegmentCenter(GenericPosture posture, CalibrationHelper calibrationHelper, GenericPostureImpactSegmentCenter impact)
{
// The point is moved so that it stays at the center of the specified segment.
// This should take perspective into account.
if (impact == null)
{
return;
}
PointF p1 = posture.PointList[impact.Point1];
PointF p2 = posture.PointList[impact.Point2];
PointF p1Plane = calibrationHelper.GetPoint(p1);
PointF p2Plane = calibrationHelper.GetPoint(p2);
PointF resultPlane = GeometryHelper.GetMiddlePoint(p1Plane, p2Plane);
PointF result = calibrationHelper.GetImagePoint(resultPlane);
posture.PointList[impact.PointToMove] = result;
}
private static void PrepareImpacts(GenericPosture posture, int handle)
{
foreach (GenericPostureAbstractImpact impact in posture.Handles[handle].Impacts)
{
// If there is a KeepAngle impact, we'll later need to know the current angle.
if (impact.Type == ImpactType.KeepAngle)
{
GenericPostureImpactKeepAngle impactKeepAngle = impact as GenericPostureImpactKeepAngle;
PointF origin = posture.PointList[impactKeepAngle.Origin];
PointF leg1 = posture.PointList[impactKeepAngle.Leg1];
PointF leg2 = posture.PointList[impactKeepAngle.Leg2];
if (origin == leg1 || origin == leg2)
{
continue;
}
impactKeepAngle.OldAngle = GeometryHelper.GetAngle(origin, leg1, leg2);
impactKeepAngle.OldDistance = GeometryHelper.GetDistance(origin, leg2);
}
}
}
private static bool IsActive(GenericPostureAbstractConstraint constraint, GenericPosture posture)
{
if (constraint == null)
{
return(false);
}
string value = constraint.OptionGroup;
if (string.IsNullOrEmpty(value))
{
return(true);
}
string[] keys = value.Split(new char[] { '|' });
// We only implement the "AND" logic at the moment:
// in case of multiple options on the object, they all need to be active for the object to be active.
bool active = true;
foreach (string key in keys)
{
if (!posture.Options.ContainsKey(key))
{
continue;
}
if (!posture.Options[key].Value)
{
active = false;
break;
}
}
return(active);
}