private void OnLoadModel(object sender, RoutedEventArgs e)
{
PoseNet posenet = new PoseNet();
using (var graph = new TFGraph())
{
graph.Import(File.ReadAllBytes("Resources\\frozen_model.bytes"));
var session = new TFSession(graph);
string input = "photo.jpg";
var tensor = ImageUtil.CreateTensorFromImageFile(input);
var runner = session.GetRunner();
runner.AddInput(graph["image"][0], tensor);
runner.Fetch(
graph["heatmap"][0],
graph["offset_2"][0],
graph["displacement_fwd_2"][0],
graph["displacement_bwd_2"][0]
);
var result = runner.Run();
var heatmap = (float[, , , ])result[0].GetValue(jagged: false);
var offsets = (float[, , , ])result[1].GetValue(jagged: false);
var displacementsFwd = (float[, , , ])result[2].GetValue(jagged: false);
var displacementsBwd = (float[, , , ])result[3].GetValue(jagged: false);
var poses = posenet.DecodeMultiplePoses(
heatmap, offsets,
displacementsFwd,
displacementsBwd,
outputStride: 16, maxPoseDetections: 15,
scoreThreshold: 0.001f, nmsRadius: 20);
}
}
// Use this for initialization
void Start()
{
var heatmap = GetResourcesToFloats("heatmaps", 33, 33, 17);
var offsets = GetResourcesToFloats("offsets", 33, 33, 34);
var displacementsFwd = GetResourcesToFloats("displacementsFwd", 33, 33, 32);
var displacementsBwd = GetResourcesToFloats("displacementsBwd", 33, 33, 32);
poses = posenet.DecodeMultiplePoses(
heatmap, offsets,
displacementsFwd,
displacementsBwd,
outputStride: 16, maxPoseDetections: 15,
scoreThreshold: 0.5f, nmsRadius: 20);
gl = glgo.GetComponent <GLRenderer>();
}
/// <summary>
/// Receive a frame from camera.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void Camera_ImageGrabbed(object sender, EventArgs e)
{
if (camera.Retrieve(frame))
{
CvInvoke.Flip(frame, frame, Emgu.CV.CvEnum.FlipType.Horizontal);
CvInvoke.Resize(frame, resizedFrame, new System.Drawing.Size(detectionSize, detectionSize), 0, 0);
TFTensor tensor = TransformInput(resizedFrame.Bitmap);
TFSession.Runner runner = session.GetRunner();
runner.AddInput(graph["image"][0], tensor);
runner.Fetch(
graph["heatmap"][0],
graph["offset_2"][0],
graph["displacement_fwd_2"][0],
graph["displacement_bwd_2"][0]
);
var result = runner.Run();
var heatmap = (float[, , , ])result[0].GetValue(jagged: false);
var offsets = (float[, , , ])result[1].GetValue(jagged: false);
var displacementsFwd = (float[, , , ])result[2].GetValue(jagged: false);
var displacementsBwd = (float[, , , ])result[3].GetValue(jagged: false);
Pose[] poses = posenet.DecodeMultiplePoses(
heatmap, offsets,
displacementsFwd,
displacementsBwd,
outputStride: 16, maxPoseDetections: 100,
scoreThreshold: 0.5f, nmsRadius: 20);
Drawing(frame, poses);
Dispatcher.Invoke(new Action(() =>
{
img.Source = frame.Bitmap.BitmapToBitmapSource();
}));
}
}
void Start()
{
TextAsset graphModel = Resources.Load("frozen_model") as TextAsset;
var graph = new TFGraph();
graph.Import(graphModel.bytes);
var session = new TFSession(graph);
Texture2D image = Resources.Load("tennis_in_crowd") as Texture2D;
image = scaled(image, ImageSize, ImageSize);
var tensor = TransformInput(image.GetPixels32(), ImageSize, ImageSize);
var runner = session.GetRunner();
runner.AddInput(graph ["image"] [0], tensor);
runner.Fetch(
graph ["heatmap"] [0],
graph ["offset_2"] [0],
graph ["displacement_fwd_2"] [0],
graph ["displacement_bwd_2"] [0]
);
var result = runner.Run();
var heatmap = (float [, , , ])result [0].GetValue(jagged: false);
var offsets = (float [, , , ])result [1].GetValue(jagged: false);
var displacementsFwd = (float [, , , ])result [2].GetValue(jagged: false);
var displacementsBwd = (float [, , , ])result [3].GetValue(jagged: false);
//Debug.Log(mean(heatmap));
poses = posenet.DecodeMultiplePoses(
heatmap, offsets,
displacementsFwd,
displacementsBwd,
outputStride: 16, maxPoseDetections: 15,
scoreThreshold: 0.5f, nmsRadius: 20);
gl = glgo.GetComponent <GLRenderer>();
}
IEnumerator PoseUpdate(Texture2D texture)
{
texture = scaled(texture, ImageSize, ImageSize);
var tensor = TransformInput(texture.GetPixels32(), ImageSize, ImageSize);
var runner = session.GetRunner();
runner.AddInput(graph["image"][0], tensor);
runner.Fetch(
graph["heatmap"][0],
graph["offset_2"][0],
graph["displacement_fwd_2"][0],
graph["displacement_bwd_2"][0]
);
var result = runner.Run();
var heatmap = (float[, , , ])result[0].GetValue(jagged: false);
var offsets = (float[, , , ])result[1].GetValue(jagged: false);
var displacementsFwd = (float[, , , ])result[2].GetValue(jagged: false);
var displacementsBwd = (float[, , , ])result[3].GetValue(jagged: false);
// Debug.Log(PoseNet.mean(heatmap));
poses = posenet.DecodeMultiplePoses(
heatmap, offsets,
displacementsFwd,
displacementsBwd,
outputStride: 16, maxPoseDetections: 15,
scoreThreshold: 0.5f, nmsRadius: 20);
isPosing = false;
texture = null;
Resources.UnloadUnusedAssets();
yield return(null);
}
IEnumerator PoseUpdate(Texture2D texture)
{
if (texture.width != detectWidth || texture.height != detectHeight)
{
texture = scaled(texture, detectWidth, detectHeight);
}
var tensor = TransformInput(texture.GetPixels32());
var runner = session.GetRunner();
runner.AddInput(graph["image"][0], tensor);
runner.Fetch(
graph["heatmap"][0],
graph["offset_2"][0],
graph["displacement_fwd_2"][0],
graph["displacement_bwd_2"][0]
);
var result = runner.Run();
var heatmap = (float[, , , ])result[0].GetValue(jagged: false);
var offsets = (float[, , , ])result[1].GetValue(jagged: false);
var displacementsFwd = (float[, , , ])result[2].GetValue(jagged: false);
var displacementsBwd = (float[, , , ])result[3].GetValue(jagged: false);
// Debug.Log(PoseNet.mean(heatmap));
poses = posenet.DecodeMultiplePoses(
heatmap, offsets,
displacementsFwd,
displacementsBwd,
outputStride: 16, maxPoseDetections: 1,
scoreThreshold: 0.5f, nmsRadius: 20);
isPosing = false;
/*/ここからtakasakaのコード
* if(poses.Length > 0 && poses[0].score >= minPoseConfidence){
* var pose = poses[0];
*
* src.Clear();
* src["nose"] = pose.keypoints[0];
* src["eyeL"] = pose.keypoints[1]; src["eyeR"] = pose.keypoints[2];
* src["earL"] = pose.keypoints[3]; src["earR"] = pose.keypoints[4];
* src["upperArmL"] = pose.keypoints[5]; src["upperArmR"] = pose.keypoints[6];
* src["lowerArmL"] = pose.keypoints[7]; src["lowerArmR"] = pose.keypoints[8];
* src["handL"] = pose.keypoints[9]; src["handR"] = pose.keypoints[10];
* src["upperLegL"] = pose.keypoints[11]; src["upperLegR"] = pose.keypoints[12];
* src["lowerLegL"] = pose.keypoints[13]; src["lowerLegR"] = pose.keypoints[14];
* src["footL"] = pose.keypoints[15]; src["footR"] = pose.keypoints[16];
*
* joint.Clear();
* foreach(KeyValuePair<string, PoseNet.Keypoint> pair in src){
* if(pair.Value.score < minPartConfidence){ continue; }
*
* //PoseNetの機能強化あるいは他の姿勢推定ライブラリに切り替えられるようVector3で作っておく
* joint[pair.Key] = new Vector3(pair.Value.position.x, videoHeight - pair.Value.position.y, 0);
* }
*
* //前フレームのジョイント位置と平均を取る
* if(jointsAvg.Count == 0){
* for(int i = 0; i < jointsCount; i++){
* jointsAvg[i] = new Dictionary<string, Vector3>(joint);
* }
* }
* jointsAvg[jointsIndex++ % jointsCount] = new Dictionary<string, Vector3>(joint);
*
* foreach(string key in src.Keys){
* if(!joint.ContainsKey(key)){ continue; }
* joint[key] = Vector3.zero;
*
* int hit = 0;
* for(int i = 0; i < jointsCount; i++){
* if(!jointsAvg[i].ContainsKey(key)){ continue; }
*
* joint[key] += jointsAvg[i][key];
++hit;
* }
* if(hit > 0){ joint[key] /= hit; }
* }
*
* //左腕
* if(joint.ContainsKey("upperArmL")){
* if(joint.ContainsKey("lowerArmL")){
* if(joint.ContainsKey("upperArmR")){
* UpdateJoint(joint, "upperArmR", "upperArmL", "lowerArmL", dst["upperArmL"]);
* AdjJoint(0, -40, 0, dst["upperArmL"]); //ダミー
* }
*
* if(joint.ContainsKey("handL")){
* UpdateJoint(joint, "upperArmL", "lowerArmL", "handL", dst["lowerArmL"]);
*
* //常に手のひらを向けるよう補正
* var armLow2Hand = (joint["handL"] - joint["lowerArmL"]);
* armLow2Hand.Normalize();
* var angleX = Rad2Deg(armLow2Hand.y) + 90;
* var angleY = Mathf.Min(0, Rad2Deg(armLow2Hand.x));
* AdjJoint(angleX, angleY, 0, dst["lowerArmL"]);
*
* dst["handL"].transform.localRotation = new Quaternion();
* AdjJoint(0, 0, -20, dst["handL"]); //ダミー
* }
* }
* }
*
* //右腕
* if(joint.ContainsKey("upperArmR")){
* if(joint.ContainsKey("lowerArmR")){
* if(joint.ContainsKey("upperArmL")){
* UpdateJoint(joint, "upperArmL", "upperArmR", "lowerArmR", dst["upperArmR"]);
* AdjJoint(0, 40, 0, dst["upperArmR"]); //ダミー
* }
* if(joint.ContainsKey("handR")){
* UpdateJoint(joint, "upperArmR", "lowerArmR", "handR", dst["lowerArmR"]);
*
* //常に手のひらを向けるよう補正
* var armLow2Hand = joint["handR"]- joint["lowerArmR"];
* armLow2Hand.Normalize();
* var angleX = Rad2Deg(armLow2Hand.y) + 90;
* var angleY = Mathf.Max(0, Rad2Deg(armLow2Hand.x));
* AdjJoint(angleX, angleY, 0, dst["lowerArmR"]);
*
* dst["handR"].transform.localRotation = new Quaternion();
* AdjJoint(0, 0, 20, dst["handR"]); //ダミー
* }
* }
* }
* //胸
* if(joint.ContainsKey("upperArmL") && joint.ContainsKey("upperArmR")){
* joint["upperArmLL"] = joint["upperArmL"] + vecX;
* UpdateJoint(joint, "upperArmLL", "upperArmL", "upperArmR", dst["upperChest"], -20, 20);
* }
* //腰
* if(joint.ContainsKey("upperLegL") && joint.ContainsKey("upperLegR")){
* //基準点が無いので左肩の左水平方向に仮のジョイントを作る
* joint["upperLegLL"] = joint["upperLegL"] + vecX;
* UpdateJoint(joint, "upperLegLL", "upperLegL", "upperLegR", dst["spine"], -10, 10);
*
* float addX = -3.0f;
* float mulX = 10.0f;
*
* var pos = joint["upperLegL"] + joint["upperLegR"];
* pos /= 2;
* var x = -(pos.x - (videoWidth / 2)) / videoWidth;
*
* Vector3 tmp = dst["hips"].transform.position;
* dst["hips"].transform.position = new Vector3(x * mulX + addX, tmp.y, tmp.z);
*
* //AdjJoint(-20, 0, 0, dst["spine"]); //ダミー
* }
* //左脚
* if(joint.ContainsKey("upperLegL")){
* if(joint.ContainsKey("lowerLegL")){
* if(joint.ContainsKey("upperLegR")){
* //基準点が無いので左脚付け根の上方向に仮のジョイントを作る
* joint["upperLegLUp"] = joint["upperLegR"] - joint["upperLegL"];
* joint["upperLegLUp"].Normalize();
* joint["upperLegLUp"] = Quaternion.AngleAxis(Rad2Deg(-halfPi), vecZ) * joint["upperLegLUp"];
* joint["upperLegLUp"] += joint["upperLegL"];
* UpdateJoint(joint, "upperLegLUp", "upperLegL", "lowerLegL", dst["upperLegL"], -20, 20);
* }
* if(joint.ContainsKey("footL")){
* UpdateJoint(joint, "upperLegL", "lowerLegL", "footL", dst["lowerLegL"], -20, 20);
*
* //基準点が無いので左足首の下垂直方向に仮のジョイントを作る
* joint["footLDown"] = joint["footL"] - vecY;
* UpdateJoint(joint, "lowerLegL", "footL", "footLDown", dst["footL"]);
* }else{
* //基準点が無いので左膝の下垂直方向に仮のジョイントを作る
* joint["lowerLegLDown"] = joint["lowerLegL"] - vecY;
* UpdateJoint(joint, "upperLegL", "lowerLegL", "lowerLegLDown", dst["lowerLegL"]);
* UpdateJoint(joint, "lowerLegL", "lowerLegLDown", "lowerLegLDown", dst["footL"]);
* }
* AdjJoint(0, 10, 0, dst["footL"]); //ダミー
* }
* }
* //右脚
* if(joint.ContainsKey("upperLegR")){
* if(joint.ContainsKey("lowerLegR")){
* if(joint.ContainsKey("upperLegL")){
* //基準点が無いので右脚付け根の上方向に仮のジョイントを作る
* joint["upperLegRUp"] = joint["upperLegL"] - joint["upperLegR"];
* joint["upperLegRUp"].Normalize();
* joint["upperLegRUp"] = Quaternion.AngleAxis(Rad2Deg(halfPi), vecZ) * joint["upperLegRUp"];
* joint["upperLegRUp"] += joint["upperLegR"];
* UpdateJoint(joint, "upperLegRUp", "upperLegR", "lowerLegR", dst["upperLegR"], -20, 20);
* }
* if(joint.ContainsKey("footR")){
* UpdateJoint(joint, "upperLegR", "lowerLegR", "footR", dst["lowerLegR"], -20, 20);
*
* //基準点が無いので右足首の下垂直方向に仮のジョイントを作る
* joint["footRDown"] = joint["footR"] - vecY;
* UpdateJoint(joint, "lowerLegR", "footR", "footRDown", dst["footR"]);
* }else{
* //基準点が無いので右膝の下垂直方向に仮のジョイントを作る
* joint["lowerLegRDown"] = joint["lowerLegR"] - vecY;
* UpdateJoint(joint, "upperLegR", "lowerLegR", "lowerLegRDown", dst["lowerLegR"]);
* UpdateJoint(joint, "lowerLegR", "lowerLegRDown", "lowerLegRDown", dst["footR"]);
* }
* AdjJoint(0, -10, 0, dst["footR"]); //ダミー
* }
* }
* }
* //ここまでtakasakaのコード*/
Debug.Log("pose update:" + Time.time + ", time:" + (Time.time - nowTime) + ", FPS:" + 1 / (Time.time - nowTime));
nowTime = Time.time;
yield return(null);
}