public unsafe void ProcessARFrame(ARSession session, ARFrame frame)
{
var arcamera = frame?.Camera;
var transform = arcamera.Transform;
var viewportSize = new CoreGraphics.CGSize(Application.Graphics.Width, Application.Graphics.Height);
float near = 0.001f;
float far = 1000f;
var prj = arcamera.GetProjectionMatrix(Orientation.Value, viewportSize, near, far);
var dt = frame.GetDisplayTransform(Orientation.Value, viewportSize);
var urhoProjection = *(Matrix4 *)(void *)&prj;
urhoProjection.M43 /= 2f;
urhoProjection.M33 = far / (far - near);
urhoProjection.M34 *= -1;
//prj.M13 = 0; //center of projection
//prj.M23 = 0;
//urhoProjection.Row2 *= -1;
urhoProjection.Transpose();
Camera.SetProjection(urhoProjection);
ApplyOpenTkTransform(Camera.Node, transform);
if (!yuvTexturesInited)
{
var img = frame.CapturedImage;
// texture for UV-plane;
cameraUVtexture = new Texture2D();
cameraUVtexture.SetNumLevels(1);
cameraUVtexture.SetSize((int)img.GetWidthOfPlane(1), (int)img.GetHeightOfPlane(1), Graphics.LuminanceAlphaFormat, TextureUsage.Dynamic);
cameraUVtexture.FilterMode = TextureFilterMode.Bilinear;
cameraUVtexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
cameraUVtexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
cameraUVtexture.Name = nameof(cameraUVtexture);
Application.ResourceCache.AddManualResource(cameraUVtexture);
// texture for Y-plane;
cameraYtexture = new Texture2D();
cameraYtexture.SetNumLevels(1);
cameraYtexture.FilterMode = TextureFilterMode.Bilinear;
cameraYtexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
cameraYtexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
cameraYtexture.SetSize((int)img.Width, (int)img.Height, Graphics.LuminanceFormat, TextureUsage.Dynamic);
cameraYtexture.Name = nameof(cameraYtexture);
Application.ResourceCache.AddManualResource(cameraYtexture);
var viewport = Application.Renderer.GetViewport(0);
var videoRp = new RenderPathCommand(RenderCommandType.Quad);
videoRp.PixelShaderName = (UrhoString)ArkitShader;
videoRp.VertexShaderName = (UrhoString)ArkitShader;
videoRp.SetOutput(0, "viewport");
videoRp.SetTextureName(TextureUnit.Diffuse, cameraYtexture.Name); //sDiffMap
videoRp.SetTextureName(TextureUnit.Normal, cameraUVtexture.Name); //sNormalMap
if (Orientation != UIInterfaceOrientation.Portrait)
{
videoRp.PixelShaderDefines = new UrhoString("ARKIT_LANDSCAPE");
}
viewport.RenderPath.InsertCommand(1, videoRp);
var vrp = viewport.RenderPath.GetCommand(1);
vrp->SetShaderParameter("Tx", (float)dt.x0);
vrp->SetShaderParameter("Ty", (float)dt.y0);
vrp->SetShaderParameter("ScaleX", (float)dt.xx);
vrp->SetShaderParameter("ScaleY", (float)dt.yy);
vrp->SetShaderParameter("ScaleYX", (float)dt.yx);
vrp->SetShaderParameter("ScaleXY", (float)dt.xy);
float imageAspect = (float)img.Width / img.Height;
float yoffset;
if (ARConfiguration is ARFaceTrackingConfiguration)
{
yoffset = 0.013f;
}
else
{
yoffset = 64.0f / Math.Max(img.Width, img.Height);
}
vrp->SetShaderParameter("YOffset", yoffset);
yuvTexturesInited = true;
}
if (yuvTexturesInited)
{
UpdateBackground(frame);
}
ARFrame?.Invoke(frame);
// required!
frame.Dispose();
}
