示例#1
0
        public override unsafe void OnAttachedToNode(Node node)
        {
            CameraTexture = new Texture2D();
            CameraTexture.SetCustomTarget(GL_TEXTURE_EXTERNAL_OES);
            CameraTexture.SetNumLevels(1);
            CameraTexture.FilterMode = TextureFilterMode.Bilinear;
            CameraTexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
            CameraTexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
            CameraTexture.SetSize(Application.Graphics.Width, Application.Graphics.Height, Graphics.Float32Format, TextureUsage.Dynamic);
            CameraTexture.Name = nameof(CameraTexture);
            Application.ResourceCache.AddManualResource(CameraTexture);

            Viewport = Application.Renderer.GetViewport(0);

            var videoRp = new RenderPathCommand(RenderCommandType.Quad);

            videoRp.PixelShaderName  = (UrhoString)"ARCore";            //see CoreData/Shaders/GLSL/ARCore.glsl
            videoRp.VertexShaderName = (UrhoString)"ARCore";
            videoRp.SetOutput(0, "viewport");
            videoRp.SetTextureName(TextureUnit.Diffuse, CameraTexture.Name);
            Viewport.RenderPath.InsertCommand(1, videoRp);

            var activity = (Activity)Urho.Application.CurrentWindow.Target;

            activity.RunOnUiThread(() =>
            {
                var cameraAllowed = activity.CheckSelfPermission(Manifest.Permission.Camera);
                if (cameraAllowed != Permission.Granted)
                {
                    throw new InvalidOperationException("Camera permission: Denied");
                }

                var session = new Session(activity);
                session.SetCameraTextureName((int)CameraTexture.AsGPUObject().GPUObjectName);
                session.SetDisplayGeometry((int)SurfaceOrientation.Rotation0 /*windowManager.DefaultDisplay.Rotation*/,
                                           Application.Graphics.Width, Application.Graphics.Height);

                Config = new Config(session);
                Config.SetLightEstimationMode(Config.LightEstimationMode.AmbientIntensity);
                //Config.SetUpdateMode(Config.UpdateMode.LatestCameraImage);
                Config.SetPlaneFindingMode(Config.PlaneFindingMode.Horizontal);
                ConfigRequested?.Invoke(Config);
                if (!session.IsSupported(Config))
                {
                    throw new Exception("AR is not supported on this device with given config");
                }

                paused = false;
                session.Resume();
                Session = session;
            });

            Application.Paused  += OnPause;
            Application.Resumed += OnResume;
        }
示例#2
0
        public override unsafe void OnAttachedToNode(Node node)
        {
            CameraTexture = new Texture2D();
            CameraTexture.SetCustomTarget(GL_TEXTURE_EXTERNAL_OES);
            CameraTexture.SetNumLevels(1);
            CameraTexture.FilterMode = TextureFilterMode.Bilinear;
            CameraTexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
            CameraTexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
            CameraTexture.SetSize(Application.Graphics.Width, Application.Graphics.Height, Graphics.Float32Format, TextureUsage.Dynamic);
            CameraTexture.Name = nameof(CameraTexture);
            Application.ResourceCache.AddManualResource(CameraTexture);

            Viewport = Application.Renderer.GetViewport(0);

            var videoRp = new RenderPathCommand(RenderCommandType.Quad);

            videoRp.PixelShaderName  = (UrhoString)"ARCore";
            videoRp.VertexShaderName = (UrhoString)"ARCore";
            videoRp.SetOutput(0, "viewport");
            videoRp.SetTextureName(TextureUnit.Diffuse, CameraTexture.Name);
            Viewport.RenderPath.InsertCommand(1, videoRp);

            var activity = (Activity)Urho.Application.CurrentWindow.Target;

            activity.RunOnUiThread(() =>
            {
                var session = new Session(activity);
                session.SetCameraTextureName((int)CameraTexture.AsGPUObject().GPUObjectName);
                session.SetDisplayGeometry(Application.Graphics.Width, Application.Graphics.Height);

                if (Config == null)
                {
                    Config = Config.CreateDefaultConfig();
                    Config.SetLightingMode(Config.LightingMode.AmbientIntensity);
                    //Config.SetUpdateMode(Config.UpdateMode.LatestCameraImage);
                    Config.SetPlaneFindingMode(Config.PlaneFindingMode.Horizontal);
                }
                paused = false;
                //TODO: check Camera permissions?
                session.Resume(Config);
                Session = session;
            });

            Application.Paused  += OnPause;
            Application.Resumed += OnResume;
        }
示例#3
0
        unsafe void CreateTextures(
            int yWidth, int yHeight,
            int uvWidth, int uvHeight,
            int nativeBoundsWidth, int nativeBoundsHeight)
        {
            var cache = Application.ResourceCache;

            // texture for Y-plane;
            CameraYtexture = new Texture2D();
            CameraYtexture.SetNumLevels(1);
            CameraYtexture.FilterMode = TextureFilterMode.Bilinear;
            CameraYtexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
            CameraYtexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
            CameraYtexture.SetSize(yWidth, yHeight, Graphics.LuminanceFormat, TextureUsage.Dynamic);
            CameraYtexture.Name = nameof(CameraYtexture);
            cache.AddManualResource(CameraYtexture);

            // texture for UV-plane;
            CameraUVtexture = new Texture2D();
            CameraUVtexture.SetNumLevels(1);
            CameraUVtexture.SetSize(uvWidth, uvHeight, Graphics.LuminanceAlphaFormat, TextureUsage.Dynamic);
            CameraUVtexture.FilterMode = TextureFilterMode.Bilinear;
            CameraUVtexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
            CameraUVtexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
            CameraUVtexture.Name = nameof(CameraUVtexture);
            cache.AddManualResource(CameraUVtexture);

            float imageAspect  = (float)yWidth / yHeight;
            float screenAspect = (float)nativeBoundsHeight / nativeBoundsWidth;

            var rpc = new RenderPathCommand(RenderCommandType.Quad);

            rpc.SetTextureName(TextureUnit.Diffuse, CameraYtexture.Name);
            rpc.SetTextureName(TextureUnit.Normal, CameraUVtexture.Name);
            rpc.Type             = RenderCommandType.Quad;
            rpc.VertexShaderName = (UrhoString)"ARKit";
            rpc.PixelShaderName  = (UrhoString)"ARKit";
            rpc.SetOutput(0, "viewport");
            rpc.SetShaderParameter("CameraScale", screenAspect / imageAspect);
            RenderPathCommand = rpc;

            Application.Renderer.GetViewport(0).RenderPath.InsertCommand(1, rpc);
        }
        // https://github.com/xamarin/urho/blob/master/Extensions/Urho.Extensions.Droid.ARCore/ARCore.cs
        // https://github.com/xamarin/urho/blob/master/Bindings/Portable/SharpReality/YuvVideo.cs
        // https://stackoverflow.com/questions/29947225/access-preview-frame-from-mediacapture
        private void SetupCameraPreview()
        {
            if (camera == null)
            {
                return;
            }

            cameraTexture = new Texture2D();
            cameraTexture.SetNumLevels(1);
            cameraTexture.FilterMode = TextureFilterMode.Bilinear;
            cameraTexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
            cameraTexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
            cameraTexture.Name = nameof(cameraTexture);
            Application.Current.ResourceCache.AddManualResource(cameraTexture);

            cameraRenderPathCommand = new RenderPathCommand(RenderCommandType.Quad);
            cameraRenderPathCommand.VertexShaderName = (UrhoString)"VideoRect";
            cameraRenderPathCommand.PixelShaderName  = (UrhoString)"VideoRect";
            cameraRenderPathCommand.SetOutput(0, "viewport");
            cameraRenderPathCommand.SetTextureName(TextureUnit.Diffuse, cameraTexture.Name);
            var viewport = Application.Current.Renderer.GetViewport(0);

            viewport.RenderPath.InsertCommand(1, cameraRenderPathCommand);

            camera.FrameReady += async(frame) => {
                if (cameraTexture.Width != frame.bitmap.PixelWidth || cameraTexture.Height != frame.bitmap.PixelHeight)
                {
                    cameraTexture.SetSize(frame.bitmap.PixelWidth, frame.bitmap.PixelHeight, Graphics.RGBAFormat, TextureUsage.Dynamic);
                }

                unsafe {
                    using (var buffer = frame.bitmap.LockBuffer(Windows.Graphics.Imaging.BitmapBufferAccessMode.Read)) {
                        using (var bufferRef = buffer.CreateReference()) {
                            byte *data;
                            uint  capacity;
                            ((IMemoryBufferByteAccess)bufferRef).GetBuffer(out data, out capacity);

                            cameraTexture.SetData(0, 0, 0, frame.bitmap.PixelWidth, frame.bitmap.PixelHeight, data);
                        }
                    }
                }
            };
        }
示例#5
0
        public unsafe void ProcessARFrame(ARSession session, ARFrame frame)
        {
            var arcamera  = frame?.Camera;
            var transform = arcamera.Transform;

            var   viewportSize = new CoreGraphics.CGSize(Application.Graphics.Width, Application.Graphics.Height);
            float near         = 0.001f;
            float far          = 1000f;
            var   prj          = arcamera.GetProjectionMatrix(Orientation.Value, viewportSize, near, far);
            var   dt           = frame.GetDisplayTransform(Orientation.Value, viewportSize);

            var urhoProjection = *(Matrix4 *)(void *)&prj;

            urhoProjection.M43 /= 2f;
            urhoProjection.M33  = far / (far - near);
            urhoProjection.M34 *= -1;
            //prj.M13 = 0; //center of projection
            //prj.M23 = 0;
            //urhoProjection.Row2 *= -1;
            urhoProjection.Transpose();

            Camera.SetProjection(urhoProjection);
            ApplyOpenTkTransform(Camera.Node, transform);

            if (!yuvTexturesInited)
            {
                var img = frame.CapturedImage;

                // texture for UV-plane;
                cameraUVtexture = new Texture2D();
                cameraUVtexture.SetNumLevels(1);
                cameraUVtexture.SetSize((int)img.GetWidthOfPlane(1), (int)img.GetHeightOfPlane(1), Graphics.LuminanceAlphaFormat, TextureUsage.Dynamic);
                cameraUVtexture.FilterMode = TextureFilterMode.Bilinear;
                cameraUVtexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
                cameraUVtexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
                cameraUVtexture.Name = nameof(cameraUVtexture);
                Application.ResourceCache.AddManualResource(cameraUVtexture);

                // texture for Y-plane;
                cameraYtexture = new Texture2D();
                cameraYtexture.SetNumLevels(1);
                cameraYtexture.FilterMode = TextureFilterMode.Bilinear;
                cameraYtexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
                cameraYtexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
                cameraYtexture.SetSize((int)img.Width, (int)img.Height, Graphics.LuminanceFormat, TextureUsage.Dynamic);
                cameraYtexture.Name = nameof(cameraYtexture);
                Application.ResourceCache.AddManualResource(cameraYtexture);

                var viewport = Application.Renderer.GetViewport(0);

                var videoRp = new RenderPathCommand(RenderCommandType.Quad);
                videoRp.PixelShaderName  = (UrhoString)ArkitShader;
                videoRp.VertexShaderName = (UrhoString)ArkitShader;
                videoRp.SetOutput(0, "viewport");
                videoRp.SetTextureName(TextureUnit.Diffuse, cameraYtexture.Name);                 //sDiffMap
                videoRp.SetTextureName(TextureUnit.Normal, cameraUVtexture.Name);                 //sNormalMap

                if (Orientation != UIInterfaceOrientation.Portrait)
                {
                    videoRp.PixelShaderDefines = new UrhoString("ARKIT_LANDSCAPE");
                }

                viewport.RenderPath.InsertCommand(1, videoRp);

                var vrp = viewport.RenderPath.GetCommand(1);
                vrp->SetShaderParameter("Tx", (float)dt.x0);
                vrp->SetShaderParameter("Ty", (float)dt.y0);
                vrp->SetShaderParameter("ScaleX", (float)dt.xx);
                vrp->SetShaderParameter("ScaleY", (float)dt.yy);
                vrp->SetShaderParameter("ScaleYX", (float)dt.yx);
                vrp->SetShaderParameter("ScaleXY", (float)dt.xy);

                float imageAspect = (float)img.Width / img.Height;

                float yoffset;
                if (ARConfiguration is ARFaceTrackingConfiguration)
                {
                    yoffset = 0.013f;
                }
                else
                {
                    yoffset = 64.0f / Math.Max(img.Width, img.Height);
                }
                vrp->SetShaderParameter("YOffset", yoffset);

                yuvTexturesInited = true;
            }

            if (yuvTexturesInited)
            {
                UpdateBackground(frame);
            }

            ARFrame?.Invoke(frame);

            // required!
            frame.Dispose();
        }
示例#6
0
        public Task Run(Camera camera = null)
        {
            if (CameraTexture != null)
            {
                throw new InvalidOperationException("ARCore component is already initialized, if you want to re-configure ARCore session - use Session property.");
            }

            Camera = camera;
            if (Camera == null)
            {
                Camera = base.Scene.GetComponent <Camera>(true);
            }

            if (Camera == null)
            {
                throw new InvalidOperationException("Camera component was not found.");
            }

            CameraTexture = new Texture2D();
            CameraTexture.SetCustomTarget(GL_TEXTURE_EXTERNAL_OES);
            CameraTexture.SetNumLevels(1);
            CameraTexture.FilterMode = TextureFilterMode.Bilinear;
            CameraTexture.SetAddressMode(TextureCoordinate.U, TextureAddressMode.Clamp);
            CameraTexture.SetAddressMode(TextureCoordinate.V, TextureAddressMode.Clamp);
            CameraTexture.SetSize(Application.Graphics.Width, Application.Graphics.Height, Graphics.Float32Format, TextureUsage.Dynamic);
            CameraTexture.Name = nameof(CameraTexture);
            Application.ResourceCache.AddManualResource(CameraTexture);

            Viewport = Application.Renderer.GetViewport(0);

            if (base.Application is SimpleApplication simpleApp)
            {
                simpleApp.MoveCamera = false;
            }

            var videoRp = new RenderPathCommand(RenderCommandType.Quad);

            videoRp.PixelShaderName  = (UrhoString)ARCoreShader;            //see CoreData/Shaders/GLSL/ARCore.glsl
            videoRp.VertexShaderName = (UrhoString)ARCoreShader;
            videoRp.SetOutput(0, "viewport");
            videoRp.SetTextureName(TextureUnit.Diffuse, CameraTexture.Name);
            Viewport.RenderPath.InsertCommand(1, videoRp);

            var tcs      = new TaskCompletionSource <bool>();
            var activity = (Activity)Urho.Application.CurrentWindow.Target;

            activity.RunOnUiThread(() =>
            {
                var cameraAllowed = activity.CheckSelfPermission(Manifest.Permission.Camera);
                if (cameraAllowed != Permission.Granted)
                {
                    throw new InvalidOperationException("Camera permission: Denied");
                }

                var session = new Session(activity);
                session.SetCameraTextureName((int)CameraTexture.AsGPUObject().GPUObjectName);
                session.SetDisplayGeometry((int)SurfaceOrientation.Rotation0 /*windowManager.DefaultDisplay.Rotation*/,
                                           Application.Graphics.Width, Application.Graphics.Height);

                Config = new Config(session);
                Config.SetLightEstimationMode(Config.LightEstimationMode.AmbientIntensity);
                Config.SetUpdateMode(Config.UpdateMode.LatestCameraImage);
                Config.SetPlaneFindingMode(Config.PlaneFindingMode.Horizontal);
                ConfigRequested?.Invoke(Config);

                if (!session.IsSupported(Config))
                {
                    throw new Exception("AR is not supported on this device with given config");
                }
                session.Configure(Config);

                paused = false;
                session.Resume();
                Session = session;
                Urho.Application.InvokeOnMain(() => tcs.TrySetResult(true));
            });
            return(tcs.Task);
        }