public async Task EvaluateVideoFrameAsync(VideoFrame frame)
{
if (frame != null)
{
try
{
TimeRecorder.Restart();
ONNXModelInput inputData = new ONNXModelInput();
inputData.Data = frame;
var output = await Model.EvaluateAsync(inputData).ConfigureAwait(false);
var product = output.ClassLabel.GetAsVectorView()[0];
var loss = output.Loss[0][product];
TimeRecorder.Stop();
var lossStr = $"{(loss * 100.0f).ToString("#0.00")}%";
string timing = $"[{DateTime.Now.Hour:00}:{DateTime.Now.Minute:00}:{DateTime.Now.Second:00}] Eval took {TimeRecorder.ElapsedMilliseconds}ms";
string message = string.Empty;
if (loss >= accuracy)
{
message = $"{product} ({lossStr})\n";
}
message += timing;
message = message.Replace("\\n", "\n");
ModifyText(message);
}
catch (Exception ex)
{
var err_message = $"error: {ex.Message}";
ModifyText(err_message);
}
}
}
public async Task EvaluateVideoFrameAsync(VideoFrame frame)
{
if (frame != null)
{
try
{
ONNXModelInput inputData = new ONNXModelInput();
inputData.Data = frame;
var output = await Model.EvaluateAsync(inputData).ConfigureAwait(false);
var product = output.ClassLabel.GetAsVectorView()[0];
var loss = output.Loss[0][product];
var lossStr = string.Join(product, " " + (loss * 100.0f).ToString("#0.00") + "%");
string message = $"({DateTime.Now.Hour}:{DateTime.Now.Minute}:{DateTime.Now.Second})\n";
string prediction = $"Prediction: {product} {lossStr}";
if (loss > 0.5f)
{
message += prediction;
}
message = message.Replace("\\n", "\n");
InvokeNullsafe(new RecognitionResult(ResultType.Success, message, product, loss));
}
catch (Exception ex)
{
InvokeNullsafe(new RecognitionResult(ResultType.Failed, $"error: {ex.Message}", "", -1));
}
}
}
public async Task EvaluateVideoFrameAsync(VideoFrame frame)
{
if (frame != null)
{
try
{
TimeRecorder.Restart();
ONNXModelInput inputData = new ONNXModelInput();
inputData.Data = frame;
var output = await Model.EvaluateAsync(inputData).ConfigureAwait(false);
var product = output.ClassLabel.GetAsVectorView()[0];
var loss = output.Loss[0][product];
TimeRecorder.Stop();
var lossStr = string.Join(product, " " + (loss * 100.0f).ToString("#0.00") + "%");
string message = $"({DateTime.Now.Hour}:{DateTime.Now.Minute}:{DateTime.Now.Second})" +
$" Evaluation took {TimeRecorder.ElapsedMilliseconds}ms\n";
string prediction = $"Prediction: {product} {lossStr}";
if (loss > 0.5f)
{
message += prediction;
}
message = message.Replace("\\n", "\n");
ModifyText(message);
}
catch (Exception ex)
{
var err_message = $"error: {ex.Message}";
ModifyText(err_message);
}
}
}
public async Task EvaluateVideoFrameAsync(VideoFrame frame, VideoMediaFrame VideoFrame, SpatialCoordinateSystem worldCoordinateSystem, SpatialCoordinateSystem cameraCoordinateSystem) // <-- 2
{
if (frame != null)
{
try
{
TimeRecorder.Restart();
// A matrix to transform camera coordinate system to world coordinate system
Matrix4x4 cameraToWorld = (Matrix4x4)cameraCoordinateSystem.TryGetTransformTo(worldCoordinateSystem);
// Internal orientation of camera
CameraIntrinsics cameraIntrinsics = VideoFrame.CameraIntrinsics;
// The frame of depth camera
DepthMediaFrame depthFrame = VideoFrame.DepthMediaFrame;
// not working, cause error
// DepthCorrelatedCoordinateMapper depthFrameMapper = depthFrame.TryCreateCoordinateMapper(cameraIntrinsics, cameraCoordinateSystem);
ONNXModelInput inputData = new ONNXModelInput();
inputData.Data = frame;
var output = await Model.EvaluateAsync(inputData).ConfigureAwait(false); // <-- 3
TimeRecorder.Stop();
string timeStamp = $"({DateTime.Now})";
// $" Evaluation took {TimeRecorder.ElapsedMilliseconds}ms\n";
int count = 0;
foreach (var prediction in output)
{
var product = prediction.TagName; // <-- 4
var loss = prediction.Probability; // <-- 5
if (loss > 0.5f)
{
float left = prediction.BoundingBox.Left;
float top = prediction.BoundingBox.Top;
float right = prediction.BoundingBox.Left + prediction.BoundingBox.Width;
float bottom = prediction.BoundingBox.Top + prediction.BoundingBox.Height;
float x = prediction.BoundingBox.Left + prediction.BoundingBox.Width / 2;
float y = prediction.BoundingBox.Top + prediction.BoundingBox.Height / 2;
Direct3DSurfaceDescription pixelData = frame.Direct3DSurface.Description;
int height = pixelData.Height;
int width = pixelData.Width;
Vector3 ImageToWorld(float X, float Y)
{
// remove image distortion
// Point objectCenterPoint = cameraIntrinsics.UndistortPoint(new Point(x, y));
// screen space -> camera space
// unproject pixel coordinate of object center towards a plane that is one meter from the camera
Vector2 objectCenter = cameraIntrinsics.UnprojectAtUnitDepth(new Point(X * width, Y * height));
// construct a ray towards object
Vector3 vectorTowardsObject = Vector3.Normalize(new Vector3(objectCenter.X, objectCenter.Y, -1.0f));
// estimate the vending machine distance by its width
// less accurate than use depth frame
// magic number 940 pixels in width for an average vending machine at 2m
// float estimatedVendingMachineDepth = (0.94f / prediction.BoundingBox.Width) * 2;
float estimatedVendingMachineDepth = (0.3f / prediction.BoundingBox.Width) * 1;
// times the vector towards object by the distance to get object's vector in camera coordinate system
Vector3 vectorToObject = vectorTowardsObject * estimatedVendingMachineDepth;
// camera space -> world space
// tranform the object postion from camera coordinate system to world coordinate system
Vector3 targetPositionInWorldSpace = Vector3.Transform(vectorToObject, cameraToWorld);
return(targetPositionInWorldSpace);
}
Vector3 objectCenterInWorld = ImageToWorld(x, y);
Vector3 objectTopLeft = ImageToWorld(left, top);
Vector3 objectTopRight = ImageToWorld(right, top);
Vector3 objectBotLeft = ImageToWorld(left, bottom);
float widthInWorld = Vector3.Distance(objectTopLeft, objectTopRight);
float heightInWorld = widthInWorld / (width * prediction.BoundingBox.Width) * (height * prediction.BoundingBox.Height);
var lossStr = (loss * 100.0f).ToString("#0.00") + "%";
// lossStr = $"{prediction.BoundingBox.Width*width}X{prediction.BoundingBox.Height*height}";
UnityApp.StoreNetworkResult(timeStamp, product, lossStr, objectCenterInWorld.X, objectCenterInWorld.Y, objectCenterInWorld.Z, widthInWorld, heightInWorld);
}
}
}
catch (Exception ex)
{
var err_message = $"{ex.Message}";
ModifyText(err_message);
}
}
}