private void ReadFrame(object sender, EventArgs arg)//捕获摄像头画面的事件
{
DateTime TimeStart = DateTime.Now;
// VideoCapture捕获一帧图像
try
{
capture.Retrieve(mat, 0);
}
catch { }
// 创建Tensor作为网络输入
TFTensor tensor = Mat2Tensor(mat);
// 前向推理
TFSession.Runner runner = session.GetRunner();
runner.AddInput(graph["image_tensor"][0], tensor);
runner.Fetch(graph["num_detections"][0]);
runner.Fetch(graph["detection_scores"][0]);
runner.Fetch(graph["detection_boxes"][0]);
runner.Fetch(graph["detection_classes"][0]);
TFTensor[] outputs = runner.Run();
// 解析结果
float num = ((float[])outputs[0].GetValue(jagged: true))[0];
float[] scores = ((float[][])outputs[1].GetValue(jagged: true))[0];
float[][] boxes = ((float[][][])outputs[2].GetValue(jagged: true))[0];
float[] classes = ((float[][])outputs[3].GetValue(jagged: true))[0];
// 显示检测框和类别
for (int i = 0; i < (int)num; i++)
{
if (scores[i] > 0.8)
{
int left = (int)(boxes[i][1] * frameWidth);
int top = (int)(boxes[i][0] * frameHeight);
int right = (int)(boxes[i][3] * frameWidth);
int bottom = (int)(boxes[i][2] * frameHeight);
CvInvoke.PutText(mat, labels[(int)classes[i]] + ": " + scores[i].ToString("0.00"), new Point(left, top), FontFace.HersheyDuplex, (right - left) / 200, colorGreen);
CvInvoke.Rectangle(mat, new Rectangle(left, top, right - left, bottom - top), colorRed, 2);
}
}
// 在imageBox上显示经过缩放的图像
Mat dst = new Mat();
CvInvoke.Resize(mat, dst, new Size(imageBox.Width, imageBox.Height));
imageBox.Image = cameraFlag ? dst : null;
TimeSpan TimeCount = DateTime.Now - TimeStart;
textFPS.Text = (1000 / TimeCount.TotalMilliseconds).ToString("0.00");
}
public PolicyValue PolicyValueFn(Board board)
{
HashSet <int> legalMoves = new HashSet <int>(board.GetAvailableMoves());
float[,,,] currentState = (float[, , , ])board.CurrentState();
TFSession.Runner runner = session.GetRunner();
runner.AddInput(graph["input_states"][0], currentState);
runner.Fetch(graph["action_fc/LogSoftmax"][0], graph["evaluation_fc2/Tanh"][0]);
TFTensor[] output = runner.Run();
float evaluation = ((float[, ])(output[1].GetValue()))[0, 0];
float[,] action_fc = ((float[, ])(output[0].GetValue()));
List <ActionP> actionPs = new List <ActionP>();
for (int i = 0; i < width * width; i++)
{
if (legalMoves.Contains(i))
{
actionPs.Add(new ActionP(i, Mathf.Exp(action_fc[0, i])));
}
}
return(new PolicyValue(actionPs, evaluation));
}
public void Update(Texture2D resizedTexture, float jointDistanceLimit, float jointThreshold, Color colorThreshold, bool useLabeling)
{
Color32[] pixels = resizedTexture.GetPixels32();
TFTensor inputTensor = CreateShapes(pixels, colorThreshold);
TFSession.Runner runner = session.GetRunner();
runner.AddInput(graph["Placeholder"][0], inputTensor);
runner.Fetch(graph["split_2"][0], graph["split_2"][1], graph["split_2"][2], graph["split_2"][3]);
TFTensor[] outputTensor = runner.Run();
nnOutputPtr = outputTensor[0].Data;
nnOutputPtrX = outputTensor[1].Data;
nnOutputPtrY = outputTensor[2].Data;
nnOutputPtrZ = outputTensor[3].Data;
heatmapWidth = (int)outputTensor[0].Shape[1];
heatmapHeight = (int)outputTensor[0].Shape[2];
if (heatmapBuff == null)
{
heatmapBuff = new float[heatmapHeight, heatmapWidth, NN_JOINT_COUNT, (int)HEATMAP_TYPE.Length];
}
else
{
Array.Clear(heatmapBuff, 0, heatmapBuff.Length);
}
ExtractHeatmaps(nnOutputPtr, nnOutputPtrX, nnOutputPtrY, nnOutputPtrZ);
Extract2DJoint(jointDistanceLimit, jointThreshold, useLabeling);
Extract3DJoint();
}
static void Main(string[] args)
{
using (TFGraph graph = new TFGraph())
{
graph.Import(File.ReadAllBytes(@"C:\Users\Ben\Desktop\frozen.pb"));
TFSession session = new TFSession(graph);
TFSession.Runner runner = session.GetRunner();
float[] x1 = new float[] { 239, 958, 8, 34, 239 };
float[] x2 = new float[] { 239, 958, 8, 34, 239 };
float[] x3 = new float[] { 239, 958, 8, 34, 239 };
TFTensor x = new TFTensor(new float[][] { x1, x2, x3 });
runner.AddInput(graph["Placeholder"][0], x);
runner.Fetch(graph["add"][0]);
var output = runner.Run();
TFTensor result = output[0];
var v = result.GetValue();
return;
}
}
public ReadOnlyCollection <Prediction> Run()
{
string libsBath = ConfigurationManager.AppSettings["LibsPath"];
string setsPath = ConfigurationManager.AppSettings["SetsPath"];
string image = $@"{setsPath}\sample_flower.jpg";
byte[] model = File.ReadAllBytes($@"{setsPath}\output_graph.pb");
string[] labels = File.ReadAllLines($@"{setsPath}\output_labels.txt");
using (var graph = new TFGraph())
{
graph.Import(new TFBuffer(model));
using (var session = new TFSession(graph))
{
TFTensor tensor = ImageUtil.CreateTensorFromImageFile(image);
TFSession.Runner runner = session.GetRunner();
if (runner == null || tensor == null)
{
Console.WriteLine("Runner or Tensor is null!?");
Environment.Exit(1);
}
runner.AddInput(graph["DecodeJpeg/contents"][0], tensor);
runner.Fetch(graph["final_result"][0]);
try
{
TFTensor[] output = runner.Run();
float[,] scores = (float[, ])output[0].GetValue();
var predictions = new List <Prediction>();
for (int i = 0; i < scores.Length; i++)
{
float score = scores[0, i];
string label = labels[i];
predictions.Add(new Prediction(label, score));
}
return(predictions
.OrderByDescending(p => p.Score)
.ToList()
.AsReadOnly());
}
catch (TFException e)
{
Console.WriteLine(e.ToString());
}
}
}
return(new List <Prediction>().AsReadOnly());
}
public int RunClassifier(float[] input)
{
TFSession.Runner runner = session.GetRunner();
runner.AddInput(graph[input_name][0], input);
runner.Fetch(graph[output_name][0]);
float[,] result = runner.Run()[0].GetValue() as float[, ];
return(ArgMax(result, 0));
}
public TFTensor [] Run(params int [] inputValues)
{
Assert(inputValues.Length == inputs.Length);
session = new TFSession(graph);
runner = session.GetRunner();
for (int i = 0; i < inputs.Length; i++)
{
runner.AddInput(inputs [i], (TFTensor)inputValues [i]);
}
runner.Fetch(outputs);
return(runner.Run());
}
public override void AdjustPose()
{
TFSession.Runner runner = session.GetRunner();
runner.AddInput(graph ["input_1"] [0], new float[, ] {
{ head.rotation.x, head.rotation.y, head.rotation.z, head.rotation.w, head.position.x * scalingFactor, head.position.y * scalingFactor, head.position.z * scalingFactor,
hand_left.rotation.x, hand_left.rotation.y, hand_left.rotation.z, hand_left.rotation.w, hand_left.position.x * scalingFactor, hand_left.position.y * scalingFactor, hand_left.position.z * scalingFactor,
hand_right.rotation.x, hand_right.rotation.y, hand_right.rotation.z, hand_right.rotation.w, hand_right.position.x * scalingFactor, hand_right.position.y * scalingFactor, hand_right.position.z * scalingFactor }
});
runner.Fetch(graph["dense_2/BiasAdd"][0]);
float[,] output_tensor = runner.Run() [0].GetValue() as float[, ];
bodyTransform.rotation = new Quaternion(output_tensor [0, 0], output_tensor [0, 1], output_tensor [0, 2], output_tensor [0, 3]);
bodyTransform.position = new Vector3(output_tensor [0, 4] / scalingFactor, output_tensor [0, 5] / scalingFactor, output_tensor [0, 6] / scalingFactor);
Debug.Log(output_tensor [0, 0] + " " + output_tensor [0, 1] + " " + output_tensor [0, 2] + " " + output_tensor [0, 3] + " " + output_tensor [0, 4] + " " + output_tensor [0, 5] + " " + output_tensor [0, 6]);
}
/// <summary>
/// Receive a frame from camera.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void Camera_ImageGrabbed(object sender, EventArgs e)
{
if (camera.Retrieve(frame))
{
CvInvoke.Flip(frame, frame, Emgu.CV.CvEnum.FlipType.Horizontal);
CvInvoke.Resize(frame, resizedFrame, new System.Drawing.Size(detectionSize, detectionSize), 0, 0);
TFTensor tensor = TransformInput(resizedFrame.Bitmap);
TFSession.Runner runner = session.GetRunner();
runner.AddInput(graph["image"][0], tensor);
runner.Fetch(
graph["heatmap"][0],
graph["offset_2"][0],
graph["displacement_fwd_2"][0],
graph["displacement_bwd_2"][0]
);
var result = runner.Run();
var heatmap = (float[, , , ])result[0].GetValue(jagged: false);
var offsets = (float[, , , ])result[1].GetValue(jagged: false);
var displacementsFwd = (float[, , , ])result[2].GetValue(jagged: false);
var displacementsBwd = (float[, , , ])result[3].GetValue(jagged: false);
Pose[] poses = posenet.DecodeMultiplePoses(
heatmap, offsets,
displacementsFwd,
displacementsBwd,
outputStride: 16, maxPoseDetections: 100,
scoreThreshold: 0.5f, nmsRadius: 20);
Drawing(frame, poses);
Dispatcher.Invoke(new Action(() =>
{
img.Source = frame.Bitmap.BitmapToBitmapSource();
}));
}
}
private IList ParseYOLO(TFSession.Runner runner, float threshold, int numResultsPerClass)
{
runner.Fetch(graph["output"][0]);
var outputs = runner.Run();
var output = outputs[0].GetValue() as float[, , , ];
foreach (var o in outputs)
{
o.Dispose();
}
var gridSize = _inputWidth / _blockSize;
int numClasses = labels.Length;
var list = new List <Dictionary <string, object> >();
for (int y = 0; y < gridSize; y++)
{
for (int x = 0; x < gridSize; x++)
{
for (int b = 0; b < _numBoxesPerBlock; b++)
{
int offset = (numClasses + 5) * b;
float confidence = expit(output[0, y, x, offset + 4]);
float[] classes = new float[numClasses];
for (int c = 0; c < numClasses; c++)
{
classes[c] = output[0, y, x, offset + 5 + c];
}
softmax(classes);
int detectedClass = -1;
float maxClass = 0;
for (int c = 0; c < numClasses; ++c)
{
if (classes[c] > maxClass)
{
detectedClass = c;
maxClass = classes[c];
}
}
float confidenceInClass = maxClass * confidence;
if (confidenceInClass > threshold)
{
float xPos = (x + expit(output[0, y, x, offset + 0])) * _blockSize;
float yPos = (y + expit(output[0, y, x, offset + 1])) * _blockSize;
float w = (float)((Math.Exp(output[0, y, x, offset + 2]) * _anchors[2 * b + 0]) * _blockSize);
float h = (float)(Math.Exp(output[0, y, x, offset + 3]) * _anchors[2 * b + 1]) * _blockSize;
float xmin = Math.Max(0, (xPos - w / 2) / _inputWidth);
float ymin = Math.Max(0, (yPos - h / 2) / _inputHeight);
var rect = new Dictionary <string, float>
{
{ "x", xmin },
{ "y", ymin },
{ "w", Math.Min(1 - xmin, w / _inputWidth) },
{ "h", Math.Min(1 - ymin, h / _inputHeight) }
};
var result = new Dictionary <string, object>
{
{ "rect", rect },
{ "confidenceInClass", confidenceInClass },
{ "detectedClass", labels[detectedClass] }
};
list.Add(result);
}
}
}
}
var sortedList = list.OrderByDescending(i => i["confidenceInClass"]).ToList();
var results = new List <Dictionary <string, object> >();
var counters = new Dictionary <string, int>();
sortedList.ForEach(i =>
{
String detectedClass = (string)i["detectedClass"];
if (counters.ContainsKey(detectedClass))
{
if (counters[detectedClass] >= numResultsPerClass)
{
return;
}
counters[detectedClass] += 1;
}
else
{
counters.Add(detectedClass, 1);
}
results.Add(i);
});
//Utils.Log(results);
return(results);
}
private IList ParseSSD(TFSession.Runner runner, float threshold, int numResultsPerClass)
{
runner.Fetch(graph["detection_boxes"][0],
graph["detection_classes"][0],
graph["detection_scores"][0],
graph["num_detections"][0]);
var outputs = runner.Run();
var boxes = outputs[0].GetValue() as float[, , ];
var classes = outputs[1].GetValue() as float[, ];
var scores = outputs[2].GetValue() as float[, ];
var num_detections = outputs[3].GetValue() as float[];
foreach (var o in outputs)
{
o.Dispose();
}
var results = new List <Dictionary <string, object> >();
var counters = new Dictionary <string, int>();
for (int i = 0; i < (int)num_detections[0]; i++)
{
if (scores[0, i] < threshold)
{
continue;
}
string detectedClass = labels[(int)classes[0, i]];
if (counters.ContainsKey(detectedClass))
{
if (counters[detectedClass] >= numResultsPerClass)
{
continue;
}
counters[detectedClass] += 1;
}
else
{
counters.Add(detectedClass, 1);
}
float ymin = Math.Max(0, boxes[0, i, 0]);
float xmin = Math.Max(0, boxes[0, i, 1]);
float ymax = boxes[0, i, 2];
float xmax = boxes[0, i, 3];
var rect = new Dictionary <string, float>
{
{ "x", xmin },
{ "y", ymin },
{ "w", Math.Min(1 - xmin, xmax - xmin) },
{ "h", Math.Min(1 - ymin, ymax - ymin) }
};
var result = new Dictionary <string, object>
{
{ "rect", rect },
{ "confidenceInClass", scores[0, i] },
{ "detectedClass", detectedClass }
};
results.Add(result);
}
//Utils.Log(results);
return(results);
}
public override IObservable <Pose> Process(IObservable <IplImage> source)
{
return(Observable.Defer(() =>
{
TFSessionOptions options = new TFSessionOptions();
unsafe
{
byte[] GPUConfig = new byte[] { 0x32, 0x02, 0x20, 0x01 };
fixed(void *ptr = &GPUConfig[0])
{
options.SetConfig(new IntPtr(ptr), GPUConfig.Length);
}
}
var graph = new TFGraph();
var session = new TFSession(graph, options, null);
var bytes = File.ReadAllBytes(ModelFileName);
graph.Import(bytes);
IplImage temp = null;
TFTensor tensor = null;
TFSession.Runner runner = null;
var config = ConfigHelper.PoseConfig(PoseConfigFileName);
return source.Select(input =>
{
var poseScale = 1.0;
const int TensorChannels = 3;
var frameSize = input.Size;
var scaleFactor = ScaleFactor;
if (scaleFactor.HasValue)
{
poseScale = scaleFactor.Value;
frameSize.Width = (int)(frameSize.Width * poseScale);
frameSize.Height = (int)(frameSize.Height * poseScale);
poseScale = 1.0 / poseScale;
}
if (tensor == null || tensor.GetTensorDimension(1) != frameSize.Height || tensor.GetTensorDimension(2) != frameSize.Width)
{
tensor = new TFTensor(
TFDataType.Float,
new long[] { 1, frameSize.Height, frameSize.Width, TensorChannels },
frameSize.Width * frameSize.Height * TensorChannels * sizeof(float));
runner = session.GetRunner();
runner.AddInput(graph["Placeholder"][0], tensor);
runner.Fetch(graph["concat_1"][0]);
}
var frame = input;
if (frameSize != input.Size)
{
if (temp == null || temp.Size != frameSize)
{
temp = new IplImage(frameSize, input.Depth, input.Channels);
}
CV.Resize(input, temp);
frame = temp;
}
using (var image = new IplImage(frameSize, IplDepth.F32, TensorChannels, tensor.Data))
{
CV.Convert(frame, image);
}
// Run the model
var output = runner.Run();
// Fetch the results from output:
var poseTensor = output[0];
var pose = new Mat((int)poseTensor.Shape[0], (int)poseTensor.Shape[1], Depth.F32, 1, poseTensor.Data);
var result = new Pose(input);
var threshold = MinConfidence;
for (int i = 0; i < pose.Rows; i++)
{
BodyPart bodyPart;
bodyPart.Name = config[i];
bodyPart.Confidence = (float)pose.GetReal(i, 2);
if (bodyPart.Confidence < threshold)
{
bodyPart.Position = new Point2f(float.NaN, float.NaN);
}
else
{
bodyPart.Position.X = (float)(pose.GetReal(i, 1) * poseScale);
bodyPart.Position.Y = (float)(pose.GetReal(i, 0) * poseScale);
}
result.Add(bodyPart);
}
return result;
});
}));
}
public int ExecuteGraph(IEnumerable <Tensor> inputs_it, IEnumerable <Tensor> outputs_it)
{
Profiler.BeginSample("TFSharpInferenceComponent.ExecuteGraph");
Tensor[] inputs = inputs_it.ToArray();
Tensor[] outputs = outputs_it.ToArray();
// TODO: Can/should we pre-allocate that?
TFSession.Runner runner = m_session.GetRunner();
inputs.ToList().ForEach((Tensor input) =>
{
if (input.Shape.Length == 0)
{
var data = input.Data.GetValue(0);
if (input.DataType == typeof(int))
{
runner.AddInput(m_graph[input.Name][0], (int)data);
}
else
{
runner.AddInput(m_graph[input.Name][0], (float)data);
}
}
else
{
runner.AddInput(m_graph[input.Name][0], input.Data);
}
});
// TODO: better way to pre-allocate this?
outputs.ToList().ForEach(s => runner.Fetch(s.Name));
TFStatus status = new TFStatus();
Profiler.BeginSample("TFSharpInferenceComponent.ExecuteGraph.RunnerRun");
var out_tensors = runner.Run(status);
Profiler.EndSample();
if (!status.Ok)
{
Debug.LogError(status.StatusMessage);
return(-1);
}
Debug.Assert(outputs.Length == out_tensors.Length);
for (var i = 0; i < outputs.Length; ++i)
{
if (outputs[i].Shape.Length == 0)
{
// Handle scalars
outputs[i].Data = Array.CreateInstance(outputs[i].DataType, new long[1] {
1
});
outputs[i].Data.SetValue(out_tensors[i].GetValue(), 0);
}
else
{
outputs[i].Data = out_tensors[i].GetValue() as Array;
}
}
Profiler.EndSample();
// TODO: create error codes
return(0);
}
public override void AdjustPose()
{
TFSession.Runner runner = session.GetRunner();
//Assume model is trained with body x and z positions and y rotation normalized to that of the head
//Assume model is trained with right hand coordinate system
//Pivot the head and hands in the y-axis around the head such that the head's y rotation is 0
Vector3 rotationPoint = head.position;
float headRotation = head.rotation.eulerAngles.y;
float rotationAmount = headRotation;
TransformInfo pivotedHead = Math3D.PivotY(head, rotationPoint, rotationAmount);
TransformInfo pivotedHandLeft = Math3D.PivotY(hand_left, rotationPoint, rotationAmount);
TransformInfo pivotedHandRight = Math3D.PivotY(hand_right, rotationPoint, rotationAmount);
//pivoted_head.position = pivotedHead.position;
//pivoted_head.rotation = pivotedHead.rotation;
//pivoted_left.position = pivotedHandLeft.position;
//pivoted_left.rotation = pivotedHandLeft.rotation;
//pivoted_right.position = pivotedHandRight.position;
//pivoted_right.rotation = pivotedHandRight.rotation;
//Convert rotations to right hand coordinate system
//Quaternion convertedHeadRotation = Math3D.RightToLeftHand (pivotedHead.rotation.x, pivotedHead.rotation.y, pivotedHead.rotation.z, pivotedHead.rotation.w);
//Quaternion convertedLeftHandRotation = Math3D.RightToLeftHand (pivotedHead.rotation.x, pivotedHead.rotation.y, pivotedHead.rotation.z, pivotedHead.rotation.w);
//Quaternion convertedRightHandRotation = Math3D.RightToLeftHand (pivotedHead.rotation.x, pivotedHead.rotation.y, pivotedHead.rotation.z, pivotedHead.rotation.w);
Quaternion pivotedHeadRotation = pivotedHead.rotation;
Quaternion pivotedLeftHandRotation = pivotedHandLeft.rotation;
Quaternion pivotedRightHandRotation = pivotedHandRight.rotation;
float[,] inputs = new float[, ] {
{
pivotedHeadRotation.x,
pivotedHeadRotation.y,
pivotedHeadRotation.z,
pivotedHeadRotation.w,
0,
pivotedHead.position.y *scalingFactor,
0,
pivotedLeftHandRotation.x,
pivotedLeftHandRotation.y,
pivotedLeftHandRotation.z,
pivotedLeftHandRotation.w,
pivotedHandLeft.position.x *scalingFactor,
pivotedHandLeft.position.y *scalingFactor,
pivotedHandLeft.position.z *scalingFactor,
pivotedRightHandRotation.x,
pivotedRightHandRotation.y,
pivotedRightHandRotation.z,
pivotedRightHandRotation.w,
pivotedHandRight.position.x *scalingFactor,
pivotedHandRight.position.y *scalingFactor,
pivotedHandRight.position.z *scalingFactor
}
};
runner.AddInput(graph [inputLayer] [0], inputs);
runner.Fetch(graph[outputLayer][0]);
float[,] output_tensor = runner.Run() [0].GetValue() as float[, ];
float x = output_tensor [0, 0];
float y = output_tensor [0, 1];
float z = output_tensor [0, 2];
float w = 1 - x * x - y * y - z * z;
if (w < 0)
{
bodyTransform.position = head.position + positionOffset;
Quaternion newRotation = Quaternion.Euler(new Vector3(0, head.rotation.eulerAngles.y));
bodyTransform.rotation = newRotation;
return;
}
w = Mathf.Sqrt(w);
bodyTransform.rotation = Math3D.RightToLeftHand(x, y, z, w);
bodyTransform.Rotate(new Vector3(0, headRotation));
// position the body such that the base of the neck is right under the head
bodyTransform.position = bodyTransform.position + head.position - neck.position + positionOffset;
}