protected override async void Capture(bool anchor) { await Task.Delay(250); if (this.stats.imageCount + this.stats.queueLen >= this.stats.imageMax) { ImageLimitExceeded(); } m_bCaptureRunning = true; float captureStartTime = Time.realtimeSinceStartup; float uploadStartTime = Time.realtimeSinceStartup; #if PLATFORM_LUMIN && UNITY_2020_1 XRCameraImage image; #else XRCpuImage image; #endif ARCameraManager cameraManager = m_Sdk.cameraManager; var cameraSubsystem = cameraManager.subsystem; #if PLATFORM_LUMIN && UNITY_2020_1 if (cameraSubsystem != null && cameraSubsystem.TryGetLatestImage(out image)) #else if (cameraSubsystem != null && cameraSubsystem.TryAcquireLatestCpuImage(out image)) #endif { JobCaptureAsync j = new JobCaptureAsync(); j.run = (int)(m_ImageRun & 0x7FFFFFFF); j.index = m_ImageIndex++; j.anchor = anchor; if (mapperSettings.useGps) { j.latitude = m_Latitude; j.longitude = m_Longitude; j.altitude = m_Altitude; } else { j.latitude = j.longitude = j.altitude = 0.0; } Camera cam = this.mainCamera; ARHelper.GetIntrinsics(out j.intrinsics); Quaternion rot = cam.transform.rotation; Vector3 pos = cam.transform.position; ARHelper.GetRotation(ref rot); j.rotation = ARHelper.SwitchHandedness(Matrix4x4.Rotate(rot)); j.position = ARHelper.SwitchHandedness(pos); int width = image.width; int height = image.height; byte[] pixels; int channels = 1; if (mapperSettings.captureRgb) { ARHelper.GetPlaneDataRGB(out pixels, image); channels = 3; } else { ARHelper.GetPlaneData(out pixels, image); } byte[] capture = new byte[channels * width * height + 8192]; int useMatching = mapperSettings.checkConnectivity ? 1 : 0; Task <icvCaptureInfo> captureTask = Task.Run(() => { return(Core.CaptureImage(capture, capture.Length, pixels, width, height, channels, useMatching)); }); await captureTask; string path = string.Format("{0}/{1}", this.tempImagePath, System.Guid.NewGuid()); using (BinaryWriter writer = new BinaryWriter(File.OpenWrite(path))) { writer.Write(capture, 0, captureTask.Result.captureSize); } j.imagePath = path; j.encodedImage = ""; if (mapperSettings.checkConnectivity) { NotifyIfConnected(captureTask.Result); } if (m_SessionFirstImage) { m_SessionFirstImage = false; } j.OnStart += () => { uploadStartTime = Time.realtimeSinceStartup; mappingUIManager.SetProgress(0); mappingUIManager.ShowProgressBar(); }; j.OnResult += (SDKImageResult result) => { float et = Time.realtimeSinceStartup - uploadStartTime; Debug.Log(string.Format("Image uploaded successfully in {0} seconds", et)); mappingUIManager.HideProgressBar(); }; j.Progress.ProgressChanged += (s, progress) => { int value = (int)(100f * progress); //Debug.Log(string.Format("Upload progress: {0}%", value)); mappingUIManager.SetProgress(value); }; j.OnError += (e) => { mappingUIManager.HideProgressBar(); }; m_Jobs.Add(j); image.Dispose(); float elapsedTime = Time.realtimeSinceStartup - captureStartTime; Debug.Log(string.Format("Capture in {0} seconds", elapsedTime)); } m_bCaptureRunning = false; var captureButton = workspaceManager.captureButton.GetComponent <Button>(); captureButton.interactable = true; }
public LightValues(Text _text, ARCameraManager _cam) { text = _text; camManager = _cam; }
protected override IEnumerator Capture(bool anchor) { yield return(new WaitForSeconds(0.25f)); m_bCaptureRunning = true; XRCameraImage image; ARCameraManager cameraManager = m_Sdk.cameraManager; var cameraSubsystem = cameraManager.subsystem; if (cameraSubsystem != null && cameraSubsystem.TryGetLatestImage(out image)) { CoroutineJobCapture j = new CoroutineJobCapture(); j.host = this; j.run = (int)(m_ImageRun & 0xEFFFFFFF); j.index = m_ImageIndex++; j.anchor = anchor; if (useGPS) { j.latitude = m_Latitude; j.longitude = m_Longitude; j.altitude = m_Altitude; } else { j.latitude = j.longitude = j.altitude = 0.0; } Camera cam = this.mainCamera; Quaternion _q = cam.transform.rotation; Matrix4x4 r = Matrix4x4.Rotate(new Quaternion(_q.x, _q.y, -_q.z, -_q.w)); Vector3 _p = cam.transform.position; Vector3 p = new Vector3(_p.x, _p.y, -_p.z); j.rotation = r; j.position = p; j.intrinsics = ARHelper.GetIntrinsics(); j.width = image.width; j.height = image.height; if (m_RgbCapture) { ARHelper.GetPlaneDataRGB(out j.pixels, image); j.channels = 3; } else { ARHelper.GetPlaneData(out j.pixels, image); j.channels = 1; } if (m_SessionFirstImage) { m_SessionFirstImage = false; } m_Jobs.Add(j); image.Dispose(); } m_bCaptureRunning = false; var captureButton = workspaceManager.captureButton.GetComponent <Button>(); captureButton.interactable = true; }
void Awake() { m_CameraManager = GetComponent <ARCameraManager>(); }
private void Awake() { GameObject arCamera = GameObject.FindGameObjectWithTag("MainCamera"); aRCameraManger = arCamera.GetComponent <ARCameraManager>(); }
public override IEnumerator Localize() { XRCpuImage image; ARCameraManager cameraManager = m_Sdk.cameraManager; var cameraSubsystem = cameraManager.subsystem; if (cameraSubsystem != null && cameraSubsystem.TryAcquireLatestCpuImage(out image)) { stats.localizationAttemptCount++; Vector3 camPos = m_Cam.transform.position; Quaternion camRot = m_Cam.transform.rotation; byte[] pixels; Vector4 intrinsics = ARHelper.GetIntrinsics(); ARHelper.GetPlaneData(out pixels, image); image.Dispose(); Vector3 pos = Vector3.zero; Quaternion rot = Quaternion.identity; float startTime = Time.realtimeSinceStartup; Task <int> t = Task.Run(() => { return(Immersal.Core.LocalizeImage(out pos, out rot, image.width, image.height, ref intrinsics, pixels)); }); while (!t.IsCompleted) { yield return(null); } int mapId = t.Result; if (mapId > 0 && ARSpace.mapIdToOffset.ContainsKey(mapId)) { if (mapId != lastLocalizedMapId) { if (m_ResetOnMapChange) { foreach (KeyValuePair <Transform, SpaceContainer> item in ARSpace.transformToSpace) { item.Value.filter.ResetFiltering(); } } lastLocalizedMapId = mapId; OnMapChanged?.Invoke(mapId); } ARHelper.GetRotation(ref rot); MapOffset mo = ARSpace.mapIdToOffset[mapId]; float elapsedTime = Time.realtimeSinceStartup - startTime; Debug.Log(string.Format("Relocalised in {0} seconds", elapsedTime)); stats.localizationSuccessCount++; Matrix4x4 offsetNoScale = Matrix4x4.TRS(mo.position, mo.rotation, Vector3.one); Vector3 scaledPos = new Vector3 ( pos.x * mo.scale.x, pos.y * mo.scale.y, pos.z * mo.scale.z ); Matrix4x4 cloudSpace = offsetNoScale * Matrix4x4.TRS(scaledPos, rot, Vector3.one); Matrix4x4 trackerSpace = Matrix4x4.TRS(camPos, camRot, Vector3.one); Matrix4x4 m = trackerSpace * (cloudSpace.inverse); mo.space.filter.RefinePose(m); LocalizerPose localizerPose; GetLocalizerPose(out localizerPose, mapId, pos, rot, m.inverse); OnPoseFound?.Invoke(localizerPose); } } yield return(StartCoroutine(base.Localize())); }
public unsafe void RequestLocalization() { if (!isReadyToLocalize) { NotificationManager.Instance.GenerateError("Error: " + "NOT READY TO LOCALIZE !!"); Debug.Log("******** NOT READY TO LOCALIZE !! *******"); return; } RemoveARAnchors(); XRCameraIntrinsics intr; ARCameraManager cameraManager = m_Sdk.cameraManager; var cameraSubsystem = cameraManager.subsystem; if (cameraSubsystem != null && cameraSubsystem.TryGetIntrinsics(out intr) && cameraManager.TryAcquireLatestCpuImage(out XRCpuImage image)) { loaderText.text = "Localizing..."; loaderPanel.SetActive(true); LocalizationRequest lr = new LocalizationRequest(); lr.cloud_Ids = cloudMaps; lr.width = image.width; lr.height = image.height; lr.channel = 3; lr.Camera_fx = intr.focalLength.x; lr.Camera_fy = intr.focalLength.y; lr.Camera_px = intr.principalPoint.x; lr.Camera_py = intr.principalPoint.y; lr.version = m_Sdk.arwaysdkversion; Vector3 camPos = ARCamera.transform.position; Quaternion camRot = ARCamera.transform.rotation; var format = TextureFormat.RGB24; if (m_Texture == null || m_Texture.width != image.width || m_Texture.height != image.height) { m_Texture = new Texture2D(image.width, image.height, format, false); } // Convert the image to format, flipping the image across the Y axis. // We can also get a sub rectangle, but we'll get the full image here. var conversionParams = new XRCpuImage.ConversionParams(image, format, XRCpuImage.Transformation.MirrorX); // Texture2D allows us write directly to the raw texture data // This allows us to do the conversion in-place without making any copies. var rawTextureData = m_Texture.GetRawTextureData <byte>(); try { image.Convert(conversionParams, new IntPtr(rawTextureData.GetUnsafePtr()), rawTextureData.Length); } finally { // We must dispose of the XRCameraImage after we're finished // with it to avoid leaking native resources. image.Dispose(); } // Apply the updated texture data to our texture m_Texture.Apply(); byte[] _bytesjpg = m_Texture.EncodeToJPG(); lr.image = Convert.ToBase64String(_bytesjpg); lr.timestamp = image.timestamp; //show requeset counts.. loc_attempts_txt.GetComponent <TMP_Text>().enabled = true; string output = JsonUtility.ToJson(lr); StartCoroutine(sendCameraImages(output, camPos, camRot)); } }
// Start is called before the first frame update private void OnEnable() { lightComponent = GetComponent <Light>(); manager = GetComponent <ARCameraManager>(); manager.frameReceived += OnCameraFrameReceived; }
public override async void LocalizeServer(SDKMapId[] mapIds) { #if PLATFORM_LUMIN XRCameraImage image; #else XRCpuImage image; #endif ARCameraManager cameraManager = m_Sdk.cameraManager; var cameraSubsystem = cameraManager.subsystem; #if PLATFORM_LUMIN if (cameraSubsystem.TryGetLatestImage(out image)) #else if (cameraSubsystem.TryAcquireLatestCpuImage(out image)) #endif { stats.localizationAttemptCount++; JobLocalizeServerAsync j = new JobLocalizeServerAsync(); byte[] pixels; Camera cam = Camera.main; Vector3 camPos = cam.transform.position; Quaternion camRot = cam.transform.rotation; Vector4 intrinsics; int channels = 1; int width = image.width; int height = image.height; ARHelper.GetIntrinsics(out intrinsics); ARHelper.GetPlaneData(out pixels, image); float startTime = Time.realtimeSinceStartup; Task <(byte[], icvCaptureInfo)> t = Task.Run(() => { byte[] capture = new byte[channels * width * height + 1024]; icvCaptureInfo info = Immersal.Core.CaptureImage(capture, capture.Length, pixels, width, height, channels); Array.Resize(ref capture, info.captureSize); return(capture, info); }); await t; j.image = t.Result.Item1; j.position = camPos; j.rotation = camRot; j.intrinsics = intrinsics; j.mapIds = mapIds; j.OnResult += async(SDKLocalizeResult result) => { float elapsedTime = Time.realtimeSinceStartup - startTime; if (result.success) { Debug.Log("*************************** On-Server Localization Succeeded ***************************"); Debug.Log(string.Format("Relocalized in {0} seconds", elapsedTime)); int mapServerId = result.map; if (mapServerId > 0 && ARSpace.mapHandleToOffset.ContainsKey(mapServerId)) { if (mapServerId != lastLocalizedMapHandle) { if (resetOnMapChange) { Reset(); } lastLocalizedMapHandle = mapServerId; OnMapChanged?.Invoke(mapServerId); } MapOffset mo = ARSpace.mapHandleToOffset[mapServerId]; stats.localizationSuccessCount++; Matrix4x4 responseMatrix = Matrix4x4.identity; responseMatrix.m00 = result.r00; responseMatrix.m01 = result.r01; responseMatrix.m02 = result.r02; responseMatrix.m03 = result.px; responseMatrix.m10 = result.r10; responseMatrix.m11 = result.r11; responseMatrix.m12 = result.r12; responseMatrix.m13 = result.py; responseMatrix.m20 = result.r20; responseMatrix.m21 = result.r21; responseMatrix.m22 = result.r22; responseMatrix.m23 = result.pz; Vector3 pos = responseMatrix.GetColumn(3); Quaternion rot = responseMatrix.rotation; ARHelper.GetRotation(ref rot); Matrix4x4 offsetNoScale = Matrix4x4.TRS(mo.position, mo.rotation, Vector3.one); Vector3 scaledPos = Vector3.Scale(pos, mo.scale); Matrix4x4 cloudSpace = offsetNoScale * Matrix4x4.TRS(scaledPos, rot, Vector3.one); Matrix4x4 trackerSpace = Matrix4x4.TRS(camPos, camRot, Vector3.one); Matrix4x4 m = trackerSpace * (cloudSpace.inverse); if (useFiltering) { mo.space.filter.RefinePose(m); } else { ARSpace.UpdateSpace(mo.space, m.GetColumn(3), m.rotation); } JobEcefAsync je = new JobEcefAsync(); je.id = mapServerId; je.OnResult += (SDKEcefResult result2) => { LocalizerPose localizerPose; LocalizerBase.GetLocalizerPose(out localizerPose, mapServerId, pos, rot, m.inverse, result2.ecef); OnPoseFound?.Invoke(localizerPose); }; await je.RunJobAsync(); if (ARSpace.mapHandleToMap.ContainsKey(mapServerId)) { ARMap map = ARSpace.mapHandleToMap[mapServerId]; map.NotifySuccessfulLocalization(mapServerId); } } else { Debug.Log(string.Format("Localization attempt failed after {0} seconds", elapsedTime)); } } else { Debug.Log("*************************** On-Server Localization Failed ***************************"); } }; await j.RunJobAsync(); image.Dispose(); } base.LocalizeServer(mapIds); }
public override async void Localize() { #if PLATFORM_LUMIN XRCameraImage image; #else XRCpuImage image; #endif ARCameraManager cameraManager = m_Sdk.cameraManager; var cameraSubsystem = cameraManager.subsystem; #if PLATFORM_LUMIN if (cameraSubsystem != null && cameraSubsystem.TryGetLatestImage(out image)) #else if (cameraSubsystem != null && cameraSubsystem.TryAcquireLatestCpuImage(out image)) #endif { stats.localizationAttemptCount++; Vector4 intrinsics; Vector3 camPos = m_Cam.transform.position; Quaternion camRot = m_Cam.transform.rotation; ARHelper.GetIntrinsics(out intrinsics); ARHelper.GetPlaneDataFast(ref m_PixelBuffer, image); if (m_PixelBuffer != IntPtr.Zero) { Vector3 pos = Vector3.zero; Quaternion rot = Quaternion.identity; float startTime = Time.realtimeSinceStartup; Task <int> t = Task.Run(() => { return(Immersal.Core.LocalizeImage(out pos, out rot, image.width, image.height, ref intrinsics, m_PixelBuffer)); }); await t; int mapHandle = t.Result; float elapsedTime = Time.realtimeSinceStartup - startTime; if (mapHandle >= 0 && ARSpace.mapHandleToOffset.ContainsKey(mapHandle)) { Debug.Log(string.Format("Relocalized in {0} seconds", elapsedTime)); stats.localizationSuccessCount++; if (mapHandle != lastLocalizedMapHandle) { if (resetOnMapChange) { Reset(); } lastLocalizedMapHandle = mapHandle; OnMapChanged?.Invoke(mapHandle); } ARHelper.GetRotation(ref rot); MapOffset mo = ARSpace.mapHandleToMap.ContainsKey(mapHandle) ? ARSpace.mapHandleToOffset[mapHandle] : null; if (mo == null) { return; } Matrix4x4 offsetNoScale = Matrix4x4.TRS(mo.position, mo.rotation, Vector3.one); Vector3 scaledPos = new Vector3 ( pos.x * mo.scale.x, pos.y * mo.scale.y, pos.z * mo.scale.z ); Matrix4x4 cloudSpace = offsetNoScale * Matrix4x4.TRS(scaledPos, rot, Vector3.one); Matrix4x4 trackerSpace = Matrix4x4.TRS(camPos, camRot, Vector3.one); Matrix4x4 m = trackerSpace * (cloudSpace.inverse); if (useFiltering) { mo.space.filter.RefinePose(m); } else { ARSpace.UpdateSpace(mo.space, m.GetColumn(3), m.rotation); } LocalizerPose localizerPose; GetLocalizerPose(out localizerPose, mapHandle, pos, rot, m.inverse); OnPoseFound?.Invoke(localizerPose); if (ARSpace.mapHandleToMap.ContainsKey(mapHandle)) { ARMap map = ARSpace.mapHandleToMap[mapHandle]; map.NotifySuccessfulLocalization(mapHandle); } } else { Debug.Log(string.Format("Localization attempt failed after {0} seconds", elapsedTime)); } } image.Dispose(); } base.Localize(); }
void Awake() { m_cameraManager = m_camera.GetComponent <ARCameraManager>(); }
void OnEnable() { cameraManager_ = FindObjectOfType <ARCameraManager>(); cameraManager_.frameReceived += onCameraFrameReceived; }
public void SetCameraManager(ARCameraManager arCameraManager) { cameraManager = arCameraManager; cameraManager.frameReceived += OnCameraFrameReceived; }
public unsafe void RequestLocalization() { if (sentrequestCount < maximumRequestinSession) { Debug.Log(" >>>>>> RequestLocalization <<<<<<< " + cloudMaps.Count); m_Sdk.GetComponent <MultiMapAssetImporter>().RemoveARAnchors(); loaderText.text = "Localizing..."; XRCameraIntrinsics intr; ARCameraManager cameraManager = m_Sdk.cameraManager; var cameraSubsystem = cameraManager.subsystem; if (cameraSubsystem != null && cameraSubsystem.TryGetIntrinsics(out intr) && cameraManager.TryAcquireLatestCpuImage(out XRCpuImage image)) { //Debug.Log("Cloud ID >>>>>>>>>>>>>>> : " + cloud_id); var format = TextureFormat.RGB24; if (m_Texture == null || m_Texture.width != image.width || m_Texture.height != image.height) { m_Texture = new Texture2D(image.width, image.height, format, false); } // Convert the image to format, flipping the image across the Y axis. // We can also get a sub rectangle, but we'll get the full image here. var conversionParams = new XRCpuImage.ConversionParams(image, format, XRCpuImage.Transformation.MirrorX); // Texture2D allows us write directly to the raw texture data // This allows us to do the conversion in-place without making any copies. var rawTextureData = m_Texture.GetRawTextureData <byte>(); try { image.Convert(conversionParams, new IntPtr(rawTextureData.GetUnsafePtr()), rawTextureData.Length); } finally { // We must dispose of the XRCameraImage after we're finished // with it to avoid leaking native resources. image.Dispose(); } // Apply the updated texture data to our texture m_Texture.Apply(); //show requeset counts.. loc_attempts_txt.GetComponent <TMP_Text>().enabled = true; loc_map_txt.GetComponent <TMP_Text>().enabled = true; byte[] _bytesjpg = m_Texture.EncodeToJPG(); loc_map_txt.text = ""; Debug.Log("TotalMaps: " + cloudMaps.Count); LocalizationRequestwithToken lr = new LocalizationRequestwithToken(); lr.developer_token = m_Sdk.developerToken; lr.cloud_Ids = cloudMaps; lr.width = image.width; lr.height = image.height; lr.channel = 3; lr.Camera_fx = intr.focalLength.x; lr.Camera_fy = intr.focalLength.y; lr.Camera_px = intr.principalPoint.x; lr.Camera_py = intr.principalPoint.y; lr.version = m_Sdk.arwaysdkversion; lr.image = Convert.ToBase64String(_bytesjpg); lr.timestamp = image.timestamp; ArCameraOffset arCameraOffset = new ArCameraOffset { position = ARCamera.transform.position, rotation = ARCamera.transform.rotation }; ArCameraPoseQueue.Enqueue(arCameraOffset); string loc_request_data = JsonUtility.ToJson(lr); SendCameraImages(loc_request_data); sentrequestCount++; } } else { CancelInvoke("RequestLocalization"); } }
/// <summary> /// The ARCameraManager is needed to acquire the frame to send to scapekit's backend /// </summary> /// <param name="arCameraManager"> /// The arCameraManager usually attached to the ARCamera /// </param> public virtual void SetCameraManager(ARCameraManager arCameraManager) { this.scapeSessionNative.SetCameraManager(arCameraManager); }