unsafe void CreateTextures(
int yWidth, int yHeight,
int uvWidth, int uvHeight,
int nativeBoundsWidth, int nativeBoundsHeight)
{
var cache = Application.ResourceCache;
// texture for Y-plane;
CameraYtexture = new Texture2D();
CameraYtexture.SetNumLevels(1);
CameraYtexture.FilterMode = TextureFilterMode.Bilinear;
CameraYtexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
CameraYtexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
CameraYtexture.SetSize(yWidth, yHeight, Graphics.LuminanceFormat, TextureUsage.Dynamic);
CameraYtexture.Name = nameof(CameraYtexture);
cache.AddManualResource(CameraYtexture);
// texture for UV-plane;
CameraUVtexture = new Texture2D();
CameraUVtexture.SetNumLevels(1);
CameraUVtexture.SetSize(uvWidth, uvHeight, Graphics.LuminanceAlphaFormat, TextureUsage.Dynamic);
CameraUVtexture.FilterMode = TextureFilterMode.Bilinear;
CameraUVtexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
CameraUVtexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
CameraUVtexture.Name = nameof(CameraUVtexture);
cache.AddManualResource(CameraUVtexture);
float imageAspect = (float)yWidth / yHeight;
float screenAspect = (float)nativeBoundsHeight / nativeBoundsWidth;
var rpc = new RenderPathCommand(RenderCommandType.Quad);
rpc.SetTextureName(TextureUnit.Diffuse, CameraYtexture.Name);
rpc.SetTextureName(TextureUnit.Normal, CameraUVtexture.Name);
rpc.Type = RenderCommandType.Quad;
rpc.VertexShaderName = (UrhoString)"ARKit";
rpc.PixelShaderName = (UrhoString)"ARKit";
rpc.SetOutput(0, "viewport");
rpc.SetShaderParameter("CameraScale", screenAspect / imageAspect);
RenderPathCommand = rpc;
Application.Renderer.GetViewport(0).RenderPath.InsertCommand(1, rpc);
}
public override unsafe void OnAttachedToNode(Node node)
{
CameraTexture = new Texture2D();
CameraTexture.SetCustomTarget(GL_TEXTURE_EXTERNAL_OES);
CameraTexture.SetNumLevels(1);
CameraTexture.FilterMode = TextureFilterMode.Bilinear;
CameraTexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
CameraTexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
CameraTexture.SetSize(Application.Graphics.Width, Application.Graphics.Height, Graphics.Float32Format, TextureUsage.Dynamic);
CameraTexture.Name = nameof(CameraTexture);
Application.ResourceCache.AddManualResource(CameraTexture);
Viewport = Application.Renderer.GetViewport(0);
var videoRp = new RenderPathCommand(RenderCommandType.Quad);
videoRp.PixelShaderName = (UrhoString)"ARCore"; //see CoreData/Shaders/GLSL/ARCore.glsl
videoRp.VertexShaderName = (UrhoString)"ARCore";
videoRp.SetOutput(0, "viewport");
videoRp.SetTextureName(TextureUnit.Diffuse, CameraTexture.Name);
Viewport.RenderPath.InsertCommand(1, videoRp);
var activity = (Activity)Urho.Application.CurrentWindow.Target;
activity.RunOnUiThread(() =>
{
var cameraAllowed = activity.CheckSelfPermission(Manifest.Permission.Camera);
if (cameraAllowed != Permission.Granted)
{
throw new InvalidOperationException("Camera permission: Denied");
}
var session = new Session(activity);
session.SetCameraTextureName((int)CameraTexture.AsGPUObject().GPUObjectName);
session.SetDisplayGeometry((int)SurfaceOrientation.Rotation0 /*windowManager.DefaultDisplay.Rotation*/,
Application.Graphics.Width, Application.Graphics.Height);
Config = new Config(session);
Config.SetLightEstimationMode(Config.LightEstimationMode.AmbientIntensity);
//Config.SetUpdateMode(Config.UpdateMode.LatestCameraImage);
Config.SetPlaneFindingMode(Config.PlaneFindingMode.Horizontal);
ConfigRequested?.Invoke(Config);
if (!session.IsSupported(Config))
{
throw new Exception("AR is not supported on this device with given config");
}
paused = false;
session.Resume();
Session = session;
});
Application.Paused += OnPause;
Application.Resumed += OnResume;
}
public override unsafe void OnAttachedToNode(Node node)
{
CameraTexture = new Texture2D();
CameraTexture.SetCustomTarget(GL_TEXTURE_EXTERNAL_OES);
CameraTexture.SetNumLevels(1);
CameraTexture.FilterMode = TextureFilterMode.Bilinear;
CameraTexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
CameraTexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
CameraTexture.SetSize(Application.Graphics.Width, Application.Graphics.Height, Graphics.Float32Format, TextureUsage.Dynamic);
CameraTexture.Name = nameof(CameraTexture);
Application.ResourceCache.AddManualResource(CameraTexture);
Viewport = Application.Renderer.GetViewport(0);
var videoRp = new RenderPathCommand(RenderCommandType.Quad);
videoRp.PixelShaderName = (UrhoString)"ARCore";
videoRp.VertexShaderName = (UrhoString)"ARCore";
videoRp.SetOutput(0, "viewport");
videoRp.SetTextureName(TextureUnit.Diffuse, CameraTexture.Name);
Viewport.RenderPath.InsertCommand(1, videoRp);
var activity = (Activity)Urho.Application.CurrentWindow.Target;
activity.RunOnUiThread(() =>
{
var session = new Session(activity);
session.SetCameraTextureName((int)CameraTexture.AsGPUObject().GPUObjectName);
session.SetDisplayGeometry(Application.Graphics.Width, Application.Graphics.Height);
if (Config == null)
{
Config = Config.CreateDefaultConfig();
Config.SetLightingMode(Config.LightingMode.AmbientIntensity);
//Config.SetUpdateMode(Config.UpdateMode.LatestCameraImage);
Config.SetPlaneFindingMode(Config.PlaneFindingMode.Horizontal);
}
paused = false;
//TODO: check Camera permissions?
session.Resume(Config);
Session = session;
});
Application.Paused += OnPause;
Application.Resumed += OnResume;
}
// https://github.com/xamarin/urho/blob/master/Extensions/Urho.Extensions.Droid.ARCore/ARCore.cs
// https://github.com/xamarin/urho/blob/master/Bindings/Portable/SharpReality/YuvVideo.cs
// https://stackoverflow.com/questions/29947225/access-preview-frame-from-mediacapture
private void SetupCameraPreview()
{
if (camera == null)
{
return;
}
cameraTexture = new Texture2D();
cameraTexture.SetNumLevels(1);
cameraTexture.FilterMode = TextureFilterMode.Bilinear;
cameraTexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
cameraTexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
cameraTexture.Name = nameof(cameraTexture);
Application.Current.ResourceCache.AddManualResource(cameraTexture);
cameraRenderPathCommand = new RenderPathCommand(RenderCommandType.Quad);
cameraRenderPathCommand.VertexShaderName = (UrhoString)"VideoRect";
cameraRenderPathCommand.PixelShaderName = (UrhoString)"VideoRect";
cameraRenderPathCommand.SetOutput(0, "viewport");
cameraRenderPathCommand.SetTextureName(TextureUnit.Diffuse, cameraTexture.Name);
var viewport = Application.Current.Renderer.GetViewport(0);
viewport.RenderPath.InsertCommand(1, cameraRenderPathCommand);
camera.FrameReady += async(frame) => {
if (cameraTexture.Width != frame.bitmap.PixelWidth || cameraTexture.Height != frame.bitmap.PixelHeight)
{
cameraTexture.SetSize(frame.bitmap.PixelWidth, frame.bitmap.PixelHeight, Graphics.RGBAFormat, TextureUsage.Dynamic);
}
unsafe {
using (var buffer = frame.bitmap.LockBuffer(Windows.Graphics.Imaging.BitmapBufferAccessMode.Read)) {
using (var bufferRef = buffer.CreateReference()) {
byte *data;
uint capacity;
((IMemoryBufferByteAccess)bufferRef).GetBuffer(out data, out capacity);
cameraTexture.SetData(0, 0, 0, frame.bitmap.PixelWidth, frame.bitmap.PixelHeight, data);
}
}
}
};
}
public unsafe void ProcessARFrame(ARSession session, ARFrame frame)
{
var arcamera = frame?.Camera;
var transform = arcamera.Transform;
var viewportSize = new CoreGraphics.CGSize(Application.Graphics.Width, Application.Graphics.Height);
float near = 0.001f;
float far = 1000f;
var prj = arcamera.GetProjectionMatrix(Orientation.Value, viewportSize, near, far);
var dt = frame.GetDisplayTransform(Orientation.Value, viewportSize);
var urhoProjection = *(Matrix4 *)(void *)&prj;
urhoProjection.M43 /= 2f;
urhoProjection.M33 = far / (far - near);
urhoProjection.M34 *= -1;
//prj.M13 = 0; //center of projection
//prj.M23 = 0;
//urhoProjection.Row2 *= -1;
urhoProjection.Transpose();
Camera.SetProjection(urhoProjection);
ApplyOpenTkTransform(Camera.Node, transform);
if (!yuvTexturesInited)
{
var img = frame.CapturedImage;
// texture for UV-plane;
cameraUVtexture = new Texture2D();
cameraUVtexture.SetNumLevels(1);
cameraUVtexture.SetSize((int)img.GetWidthOfPlane(1), (int)img.GetHeightOfPlane(1), Graphics.LuminanceAlphaFormat, TextureUsage.Dynamic);
cameraUVtexture.FilterMode = TextureFilterMode.Bilinear;
cameraUVtexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
cameraUVtexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
cameraUVtexture.Name = nameof(cameraUVtexture);
Application.ResourceCache.AddManualResource(cameraUVtexture);
// texture for Y-plane;
cameraYtexture = new Texture2D();
cameraYtexture.SetNumLevels(1);
cameraYtexture.FilterMode = TextureFilterMode.Bilinear;
cameraYtexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
cameraYtexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
cameraYtexture.SetSize((int)img.Width, (int)img.Height, Graphics.LuminanceFormat, TextureUsage.Dynamic);
cameraYtexture.Name = nameof(cameraYtexture);
Application.ResourceCache.AddManualResource(cameraYtexture);
var viewport = Application.Renderer.GetViewport(0);
var videoRp = new RenderPathCommand(RenderCommandType.Quad);
videoRp.PixelShaderName = (UrhoString)ArkitShader;
videoRp.VertexShaderName = (UrhoString)ArkitShader;
videoRp.SetOutput(0, "viewport");
videoRp.SetTextureName(TextureUnit.Diffuse, cameraYtexture.Name); //sDiffMap
videoRp.SetTextureName(TextureUnit.Normal, cameraUVtexture.Name); //sNormalMap
if (Orientation != UIInterfaceOrientation.Portrait)
{
videoRp.PixelShaderDefines = new UrhoString("ARKIT_LANDSCAPE");
}
viewport.RenderPath.InsertCommand(1, videoRp);
var vrp = viewport.RenderPath.GetCommand(1);
vrp->SetShaderParameter("Tx", (float)dt.x0);
vrp->SetShaderParameter("Ty", (float)dt.y0);
vrp->SetShaderParameter("ScaleX", (float)dt.xx);
vrp->SetShaderParameter("ScaleY", (float)dt.yy);
vrp->SetShaderParameter("ScaleYX", (float)dt.yx);
vrp->SetShaderParameter("ScaleXY", (float)dt.xy);
float imageAspect = (float)img.Width / img.Height;
float yoffset;
if (ARConfiguration is ARFaceTrackingConfiguration)
{
yoffset = 0.013f;
}
else
{
yoffset = 64.0f / Math.Max(img.Width, img.Height);
}
vrp->SetShaderParameter("YOffset", yoffset);
yuvTexturesInited = true;
}
if (yuvTexturesInited)
{
UpdateBackground(frame);
}
ARFrame?.Invoke(frame);
// required!
frame.Dispose();
}
public Task Run(Camera camera = null)
{
if (CameraTexture != null)
{
throw new InvalidOperationException("ARCore component is already initialized, if you want to re-configure ARCore session - use Session property.");
}
Camera = camera;
if (Camera == null)
{
Camera = base.Scene.GetComponent <Camera>(true);
}
if (Camera == null)
{
throw new InvalidOperationException("Camera component was not found.");
}
CameraTexture = new Texture2D();
CameraTexture.SetCustomTarget(GL_TEXTURE_EXTERNAL_OES);
CameraTexture.SetNumLevels(1);
CameraTexture.FilterMode = TextureFilterMode.Bilinear;
CameraTexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
CameraTexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
CameraTexture.SetSize(Application.Graphics.Width, Application.Graphics.Height, Graphics.Float32Format, TextureUsage.Dynamic);
CameraTexture.Name = nameof(CameraTexture);
Application.ResourceCache.AddManualResource(CameraTexture);
Viewport = Application.Renderer.GetViewport(0);
if (base.Application is SimpleApplication simpleApp)
{
simpleApp.MoveCamera = false;
}
var videoRp = new RenderPathCommand(RenderCommandType.Quad);
videoRp.PixelShaderName = (UrhoString)ARCoreShader; //see CoreData/Shaders/GLSL/ARCore.glsl
videoRp.VertexShaderName = (UrhoString)ARCoreShader;
videoRp.SetOutput(0, "viewport");
videoRp.SetTextureName(TextureUnit.Diffuse, CameraTexture.Name);
Viewport.RenderPath.InsertCommand(1, videoRp);
var tcs = new TaskCompletionSource <bool>();
var activity = (Activity)Urho.Application.CurrentWindow.Target;
activity.RunOnUiThread(() =>
{
var cameraAllowed = activity.CheckSelfPermission(Manifest.Permission.Camera);
if (cameraAllowed != Permission.Granted)
{
throw new InvalidOperationException("Camera permission: Denied");
}
var session = new Session(activity);
session.SetCameraTextureName((int)CameraTexture.AsGPUObject().GPUObjectName);
session.SetDisplayGeometry((int)SurfaceOrientation.Rotation0 /*windowManager.DefaultDisplay.Rotation*/,
Application.Graphics.Width, Application.Graphics.Height);
Config = new Config(session);
Config.SetLightEstimationMode(Config.LightEstimationMode.AmbientIntensity);
Config.SetUpdateMode(Config.UpdateMode.LatestCameraImage);
Config.SetPlaneFindingMode(Config.PlaneFindingMode.Horizontal);
ConfigRequested?.Invoke(Config);
if (!session.IsSupported(Config))
{
throw new Exception("AR is not supported on this device with given config");
}
session.Configure(Config);
paused = false;
session.Resume();
Session = session;
Urho.Application.InvokeOnMain(() => tcs.TrySetResult(true));
});
return(tcs.Task);
}