protected override async void Capture(bool anchor)
{
await Task.Delay(250);
if (this.stats.imageCount + this.stats.queueLen >= this.stats.imageMax)
{
ImageLimitExceeded();
}
m_bCaptureRunning = true;
float captureStartTime = Time.realtimeSinceStartup;
float uploadStartTime = Time.realtimeSinceStartup;
#if PLATFORM_LUMIN && UNITY_2020_1
XRCameraImage image;
#else
XRCpuImage image;
#endif
ARCameraManager cameraManager = m_Sdk.cameraManager;
var cameraSubsystem = cameraManager.subsystem;
#if PLATFORM_LUMIN && UNITY_2020_1
if (cameraSubsystem != null && cameraSubsystem.TryGetLatestImage(out image))
#else
if (cameraSubsystem != null && cameraSubsystem.TryAcquireLatestCpuImage(out image))
#endif
{
JobCaptureAsync j = new JobCaptureAsync();
j.run = (int)(m_ImageRun & 0x7FFFFFFF);
j.index = m_ImageIndex++;
j.anchor = anchor;
if (mapperSettings.useGps)
{
j.latitude = m_Latitude;
j.longitude = m_Longitude;
j.altitude = m_Altitude;
}
else
{
j.latitude = j.longitude = j.altitude = 0.0;
}
Camera cam = this.mainCamera;
ARHelper.GetIntrinsics(out j.intrinsics);
Quaternion rot = cam.transform.rotation;
Vector3 pos = cam.transform.position;
ARHelper.GetRotation(ref rot);
j.rotation = ARHelper.SwitchHandedness(Matrix4x4.Rotate(rot));
j.position = ARHelper.SwitchHandedness(pos);
int width = image.width;
int height = image.height;
byte[] pixels;
int channels = 1;
if (mapperSettings.captureRgb)
{
ARHelper.GetPlaneDataRGB(out pixels, image);
channels = 3;
}
else
{
ARHelper.GetPlaneData(out pixels, image);
}
byte[] capture = new byte[channels * width * height + 8192];
int useMatching = mapperSettings.checkConnectivity ? 1 : 0;
Task <icvCaptureInfo> captureTask = Task.Run(() =>
{
return(Core.CaptureImage(capture, capture.Length, pixels, width, height, channels, useMatching));
});
await captureTask;
string path = string.Format("{0}/{1}", this.tempImagePath, System.Guid.NewGuid());
using (BinaryWriter writer = new BinaryWriter(File.OpenWrite(path)))
{
writer.Write(capture, 0, captureTask.Result.captureSize);
}
j.imagePath = path;
j.encodedImage = "";
if (mapperSettings.checkConnectivity)
{
NotifyIfConnected(captureTask.Result);
}
if (m_SessionFirstImage)
{
m_SessionFirstImage = false;
}
j.OnStart += () =>
{
uploadStartTime = Time.realtimeSinceStartup;
mappingUIManager.SetProgress(0);
mappingUIManager.ShowProgressBar();
};
j.OnResult += (SDKImageResult result) =>
{
float et = Time.realtimeSinceStartup - uploadStartTime;
Debug.Log(string.Format("Image uploaded successfully in {0} seconds", et));
mappingUIManager.HideProgressBar();
};
j.Progress.ProgressChanged += (s, progress) =>
{
int value = (int)(100f * progress);
//Debug.Log(string.Format("Upload progress: {0}%", value));
mappingUIManager.SetProgress(value);
};
j.OnError += (e) =>
{
mappingUIManager.HideProgressBar();
};
m_Jobs.Add(j);
image.Dispose();
float elapsedTime = Time.realtimeSinceStartup - captureStartTime;
Debug.Log(string.Format("Capture in {0} seconds", elapsedTime));
}
m_bCaptureRunning = false;
var captureButton = workspaceManager.captureButton.GetComponent <Button>();
captureButton.interactable = true;
}
public LightValues(Text _text, ARCameraManager _cam)
{
text = _text;
camManager = _cam;
}
protected override IEnumerator Capture(bool anchor)
{
yield return(new WaitForSeconds(0.25f));
m_bCaptureRunning = true;
XRCameraImage image;
ARCameraManager cameraManager = m_Sdk.cameraManager;
var cameraSubsystem = cameraManager.subsystem;
if (cameraSubsystem != null && cameraSubsystem.TryGetLatestImage(out image))
{
CoroutineJobCapture j = new CoroutineJobCapture();
j.host = this;
j.run = (int)(m_ImageRun & 0xEFFFFFFF);
j.index = m_ImageIndex++;
j.anchor = anchor;
if (useGPS)
{
j.latitude = m_Latitude;
j.longitude = m_Longitude;
j.altitude = m_Altitude;
}
else
{
j.latitude = j.longitude = j.altitude = 0.0;
}
Camera cam = this.mainCamera;
Quaternion _q = cam.transform.rotation;
Matrix4x4 r = Matrix4x4.Rotate(new Quaternion(_q.x, _q.y, -_q.z, -_q.w));
Vector3 _p = cam.transform.position;
Vector3 p = new Vector3(_p.x, _p.y, -_p.z);
j.rotation = r;
j.position = p;
j.intrinsics = ARHelper.GetIntrinsics();
j.width = image.width;
j.height = image.height;
if (m_RgbCapture)
{
ARHelper.GetPlaneDataRGB(out j.pixels, image);
j.channels = 3;
}
else
{
ARHelper.GetPlaneData(out j.pixels, image);
j.channels = 1;
}
if (m_SessionFirstImage)
{
m_SessionFirstImage = false;
}
m_Jobs.Add(j);
image.Dispose();
}
m_bCaptureRunning = false;
var captureButton = workspaceManager.captureButton.GetComponent <Button>();
captureButton.interactable = true;
}
void Awake()
{
m_CameraManager = GetComponent <ARCameraManager>();
}
private void Awake()
{
GameObject arCamera = GameObject.FindGameObjectWithTag("MainCamera");
aRCameraManger = arCamera.GetComponent <ARCameraManager>();
}
public override IEnumerator Localize()
{
XRCpuImage image;
ARCameraManager cameraManager = m_Sdk.cameraManager;
var cameraSubsystem = cameraManager.subsystem;
if (cameraSubsystem != null && cameraSubsystem.TryAcquireLatestCpuImage(out image))
{
stats.localizationAttemptCount++;
Vector3 camPos = m_Cam.transform.position;
Quaternion camRot = m_Cam.transform.rotation;
byte[] pixels;
Vector4 intrinsics = ARHelper.GetIntrinsics();
ARHelper.GetPlaneData(out pixels, image);
image.Dispose();
Vector3 pos = Vector3.zero;
Quaternion rot = Quaternion.identity;
float startTime = Time.realtimeSinceStartup;
Task <int> t = Task.Run(() =>
{
return(Immersal.Core.LocalizeImage(out pos, out rot, image.width, image.height, ref intrinsics, pixels));
});
while (!t.IsCompleted)
{
yield return(null);
}
int mapId = t.Result;
if (mapId > 0 && ARSpace.mapIdToOffset.ContainsKey(mapId))
{
if (mapId != lastLocalizedMapId)
{
if (m_ResetOnMapChange)
{
foreach (KeyValuePair <Transform, SpaceContainer> item in ARSpace.transformToSpace)
{
item.Value.filter.ResetFiltering();
}
}
lastLocalizedMapId = mapId;
OnMapChanged?.Invoke(mapId);
}
ARHelper.GetRotation(ref rot);
MapOffset mo = ARSpace.mapIdToOffset[mapId];
float elapsedTime = Time.realtimeSinceStartup - startTime;
Debug.Log(string.Format("Relocalised in {0} seconds", elapsedTime));
stats.localizationSuccessCount++;
Matrix4x4 offsetNoScale = Matrix4x4.TRS(mo.position, mo.rotation, Vector3.one);
Vector3 scaledPos = new Vector3
(
pos.x * mo.scale.x,
pos.y * mo.scale.y,
pos.z * mo.scale.z
);
Matrix4x4 cloudSpace = offsetNoScale * Matrix4x4.TRS(scaledPos, rot, Vector3.one);
Matrix4x4 trackerSpace = Matrix4x4.TRS(camPos, camRot, Vector3.one);
Matrix4x4 m = trackerSpace * (cloudSpace.inverse);
mo.space.filter.RefinePose(m);
LocalizerPose localizerPose;
GetLocalizerPose(out localizerPose, mapId, pos, rot, m.inverse);
OnPoseFound?.Invoke(localizerPose);
}
}
yield return(StartCoroutine(base.Localize()));
}
public unsafe void RequestLocalization()
{
if (!isReadyToLocalize)
{
NotificationManager.Instance.GenerateError("Error: " + "NOT READY TO LOCALIZE !!");
Debug.Log("******** NOT READY TO LOCALIZE !! *******");
return;
}
RemoveARAnchors();
XRCameraIntrinsics intr;
ARCameraManager cameraManager = m_Sdk.cameraManager;
var cameraSubsystem = cameraManager.subsystem;
if (cameraSubsystem != null && cameraSubsystem.TryGetIntrinsics(out intr) && cameraManager.TryAcquireLatestCpuImage(out XRCpuImage image))
{
loaderText.text = "Localizing...";
loaderPanel.SetActive(true);
LocalizationRequest lr = new LocalizationRequest();
lr.cloud_Ids = cloudMaps;
lr.width = image.width;
lr.height = image.height;
lr.channel = 3;
lr.Camera_fx = intr.focalLength.x;
lr.Camera_fy = intr.focalLength.y;
lr.Camera_px = intr.principalPoint.x;
lr.Camera_py = intr.principalPoint.y;
lr.version = m_Sdk.arwaysdkversion;
Vector3 camPos = ARCamera.transform.position;
Quaternion camRot = ARCamera.transform.rotation;
var format = TextureFormat.RGB24;
if (m_Texture == null || m_Texture.width != image.width || m_Texture.height != image.height)
{
m_Texture = new Texture2D(image.width, image.height, format, false);
}
// Convert the image to format, flipping the image across the Y axis.
// We can also get a sub rectangle, but we'll get the full image here.
var conversionParams = new XRCpuImage.ConversionParams(image, format, XRCpuImage.Transformation.MirrorX);
// Texture2D allows us write directly to the raw texture data
// This allows us to do the conversion in-place without making any copies.
var rawTextureData = m_Texture.GetRawTextureData <byte>();
try
{
image.Convert(conversionParams, new IntPtr(rawTextureData.GetUnsafePtr()), rawTextureData.Length);
}
finally
{
// We must dispose of the XRCameraImage after we're finished
// with it to avoid leaking native resources.
image.Dispose();
}
// Apply the updated texture data to our texture
m_Texture.Apply();
byte[] _bytesjpg = m_Texture.EncodeToJPG();
lr.image = Convert.ToBase64String(_bytesjpg);
lr.timestamp = image.timestamp;
//show requeset counts..
loc_attempts_txt.GetComponent <TMP_Text>().enabled = true;
string output = JsonUtility.ToJson(lr);
StartCoroutine(sendCameraImages(output, camPos, camRot));
}
}
// Start is called before the first frame update
private void OnEnable()
{
lightComponent = GetComponent <Light>();
manager = GetComponent <ARCameraManager>();
manager.frameReceived += OnCameraFrameReceived;
}
public override async void LocalizeServer(SDKMapId[] mapIds)
{
#if PLATFORM_LUMIN
XRCameraImage image;
#else
XRCpuImage image;
#endif
ARCameraManager cameraManager = m_Sdk.cameraManager;
var cameraSubsystem = cameraManager.subsystem;
#if PLATFORM_LUMIN
if (cameraSubsystem.TryGetLatestImage(out image))
#else
if (cameraSubsystem.TryAcquireLatestCpuImage(out image))
#endif
{
stats.localizationAttemptCount++;
JobLocalizeServerAsync j = new JobLocalizeServerAsync();
byte[] pixels;
Camera cam = Camera.main;
Vector3 camPos = cam.transform.position;
Quaternion camRot = cam.transform.rotation;
Vector4 intrinsics;
int channels = 1;
int width = image.width;
int height = image.height;
ARHelper.GetIntrinsics(out intrinsics);
ARHelper.GetPlaneData(out pixels, image);
float startTime = Time.realtimeSinceStartup;
Task <(byte[], icvCaptureInfo)> t = Task.Run(() =>
{
byte[] capture = new byte[channels * width * height + 1024];
icvCaptureInfo info = Immersal.Core.CaptureImage(capture, capture.Length, pixels, width, height, channels);
Array.Resize(ref capture, info.captureSize);
return(capture, info);
});
await t;
j.image = t.Result.Item1;
j.position = camPos;
j.rotation = camRot;
j.intrinsics = intrinsics;
j.mapIds = mapIds;
j.OnResult += async(SDKLocalizeResult result) =>
{
float elapsedTime = Time.realtimeSinceStartup - startTime;
if (result.success)
{
Debug.Log("*************************** On-Server Localization Succeeded ***************************");
Debug.Log(string.Format("Relocalized in {0} seconds", elapsedTime));
int mapServerId = result.map;
if (mapServerId > 0 && ARSpace.mapHandleToOffset.ContainsKey(mapServerId))
{
if (mapServerId != lastLocalizedMapHandle)
{
if (resetOnMapChange)
{
Reset();
}
lastLocalizedMapHandle = mapServerId;
OnMapChanged?.Invoke(mapServerId);
}
MapOffset mo = ARSpace.mapHandleToOffset[mapServerId];
stats.localizationSuccessCount++;
Matrix4x4 responseMatrix = Matrix4x4.identity;
responseMatrix.m00 = result.r00; responseMatrix.m01 = result.r01; responseMatrix.m02 = result.r02; responseMatrix.m03 = result.px;
responseMatrix.m10 = result.r10; responseMatrix.m11 = result.r11; responseMatrix.m12 = result.r12; responseMatrix.m13 = result.py;
responseMatrix.m20 = result.r20; responseMatrix.m21 = result.r21; responseMatrix.m22 = result.r22; responseMatrix.m23 = result.pz;
Vector3 pos = responseMatrix.GetColumn(3);
Quaternion rot = responseMatrix.rotation;
ARHelper.GetRotation(ref rot);
Matrix4x4 offsetNoScale = Matrix4x4.TRS(mo.position, mo.rotation, Vector3.one);
Vector3 scaledPos = Vector3.Scale(pos, mo.scale);
Matrix4x4 cloudSpace = offsetNoScale * Matrix4x4.TRS(scaledPos, rot, Vector3.one);
Matrix4x4 trackerSpace = Matrix4x4.TRS(camPos, camRot, Vector3.one);
Matrix4x4 m = trackerSpace * (cloudSpace.inverse);
if (useFiltering)
{
mo.space.filter.RefinePose(m);
}
else
{
ARSpace.UpdateSpace(mo.space, m.GetColumn(3), m.rotation);
}
JobEcefAsync je = new JobEcefAsync();
je.id = mapServerId;
je.OnResult += (SDKEcefResult result2) =>
{
LocalizerPose localizerPose;
LocalizerBase.GetLocalizerPose(out localizerPose, mapServerId, pos, rot, m.inverse, result2.ecef);
OnPoseFound?.Invoke(localizerPose);
};
await je.RunJobAsync();
if (ARSpace.mapHandleToMap.ContainsKey(mapServerId))
{
ARMap map = ARSpace.mapHandleToMap[mapServerId];
map.NotifySuccessfulLocalization(mapServerId);
}
}
else
{
Debug.Log(string.Format("Localization attempt failed after {0} seconds", elapsedTime));
}
}
else
{
Debug.Log("*************************** On-Server Localization Failed ***************************");
}
};
await j.RunJobAsync();
image.Dispose();
}
base.LocalizeServer(mapIds);
}
public override async void Localize()
{
#if PLATFORM_LUMIN
XRCameraImage image;
#else
XRCpuImage image;
#endif
ARCameraManager cameraManager = m_Sdk.cameraManager;
var cameraSubsystem = cameraManager.subsystem;
#if PLATFORM_LUMIN
if (cameraSubsystem != null && cameraSubsystem.TryGetLatestImage(out image))
#else
if (cameraSubsystem != null && cameraSubsystem.TryAcquireLatestCpuImage(out image))
#endif
{
stats.localizationAttemptCount++;
Vector4 intrinsics;
Vector3 camPos = m_Cam.transform.position;
Quaternion camRot = m_Cam.transform.rotation;
ARHelper.GetIntrinsics(out intrinsics);
ARHelper.GetPlaneDataFast(ref m_PixelBuffer, image);
if (m_PixelBuffer != IntPtr.Zero)
{
Vector3 pos = Vector3.zero;
Quaternion rot = Quaternion.identity;
float startTime = Time.realtimeSinceStartup;
Task <int> t = Task.Run(() =>
{
return(Immersal.Core.LocalizeImage(out pos, out rot, image.width, image.height, ref intrinsics, m_PixelBuffer));
});
await t;
int mapHandle = t.Result;
float elapsedTime = Time.realtimeSinceStartup - startTime;
if (mapHandle >= 0 && ARSpace.mapHandleToOffset.ContainsKey(mapHandle))
{
Debug.Log(string.Format("Relocalized in {0} seconds", elapsedTime));
stats.localizationSuccessCount++;
if (mapHandle != lastLocalizedMapHandle)
{
if (resetOnMapChange)
{
Reset();
}
lastLocalizedMapHandle = mapHandle;
OnMapChanged?.Invoke(mapHandle);
}
ARHelper.GetRotation(ref rot);
MapOffset mo = ARSpace.mapHandleToMap.ContainsKey(mapHandle) ? ARSpace.mapHandleToOffset[mapHandle] : null;
if (mo == null)
{
return;
}
Matrix4x4 offsetNoScale = Matrix4x4.TRS(mo.position, mo.rotation, Vector3.one);
Vector3 scaledPos = new Vector3
(
pos.x * mo.scale.x,
pos.y * mo.scale.y,
pos.z * mo.scale.z
);
Matrix4x4 cloudSpace = offsetNoScale * Matrix4x4.TRS(scaledPos, rot, Vector3.one);
Matrix4x4 trackerSpace = Matrix4x4.TRS(camPos, camRot, Vector3.one);
Matrix4x4 m = trackerSpace * (cloudSpace.inverse);
if (useFiltering)
{
mo.space.filter.RefinePose(m);
}
else
{
ARSpace.UpdateSpace(mo.space, m.GetColumn(3), m.rotation);
}
LocalizerPose localizerPose;
GetLocalizerPose(out localizerPose, mapHandle, pos, rot, m.inverse);
OnPoseFound?.Invoke(localizerPose);
if (ARSpace.mapHandleToMap.ContainsKey(mapHandle))
{
ARMap map = ARSpace.mapHandleToMap[mapHandle];
map.NotifySuccessfulLocalization(mapHandle);
}
}
else
{
Debug.Log(string.Format("Localization attempt failed after {0} seconds", elapsedTime));
}
}
image.Dispose();
}
base.Localize();
}
void Awake()
{
m_cameraManager = m_camera.GetComponent <ARCameraManager>();
}
void OnEnable()
{
cameraManager_ = FindObjectOfType <ARCameraManager>();
cameraManager_.frameReceived += onCameraFrameReceived;
}
public void SetCameraManager(ARCameraManager arCameraManager)
{
cameraManager = arCameraManager;
cameraManager.frameReceived += OnCameraFrameReceived;
}
public unsafe void RequestLocalization()
{
if (sentrequestCount < maximumRequestinSession)
{
Debug.Log(" >>>>>> RequestLocalization <<<<<<< " + cloudMaps.Count);
m_Sdk.GetComponent <MultiMapAssetImporter>().RemoveARAnchors();
loaderText.text = "Localizing...";
XRCameraIntrinsics intr;
ARCameraManager cameraManager = m_Sdk.cameraManager;
var cameraSubsystem = cameraManager.subsystem;
if (cameraSubsystem != null && cameraSubsystem.TryGetIntrinsics(out intr) && cameraManager.TryAcquireLatestCpuImage(out XRCpuImage image))
{
//Debug.Log("Cloud ID >>>>>>>>>>>>>>> : " + cloud_id);
var format = TextureFormat.RGB24;
if (m_Texture == null || m_Texture.width != image.width || m_Texture.height != image.height)
{
m_Texture = new Texture2D(image.width, image.height, format, false);
}
// Convert the image to format, flipping the image across the Y axis.
// We can also get a sub rectangle, but we'll get the full image here.
var conversionParams = new XRCpuImage.ConversionParams(image, format, XRCpuImage.Transformation.MirrorX);
// Texture2D allows us write directly to the raw texture data
// This allows us to do the conversion in-place without making any copies.
var rawTextureData = m_Texture.GetRawTextureData <byte>();
try
{
image.Convert(conversionParams, new IntPtr(rawTextureData.GetUnsafePtr()), rawTextureData.Length);
}
finally
{
// We must dispose of the XRCameraImage after we're finished
// with it to avoid leaking native resources.
image.Dispose();
}
// Apply the updated texture data to our texture
m_Texture.Apply();
//show requeset counts..
loc_attempts_txt.GetComponent <TMP_Text>().enabled = true;
loc_map_txt.GetComponent <TMP_Text>().enabled = true;
byte[] _bytesjpg = m_Texture.EncodeToJPG();
loc_map_txt.text = "";
Debug.Log("TotalMaps: " + cloudMaps.Count);
LocalizationRequestwithToken lr = new LocalizationRequestwithToken();
lr.developer_token = m_Sdk.developerToken;
lr.cloud_Ids = cloudMaps;
lr.width = image.width;
lr.height = image.height;
lr.channel = 3;
lr.Camera_fx = intr.focalLength.x;
lr.Camera_fy = intr.focalLength.y;
lr.Camera_px = intr.principalPoint.x;
lr.Camera_py = intr.principalPoint.y;
lr.version = m_Sdk.arwaysdkversion;
lr.image = Convert.ToBase64String(_bytesjpg);
lr.timestamp = image.timestamp;
ArCameraOffset arCameraOffset = new ArCameraOffset
{
position = ARCamera.transform.position,
rotation = ARCamera.transform.rotation
};
ArCameraPoseQueue.Enqueue(arCameraOffset);
string loc_request_data = JsonUtility.ToJson(lr);
SendCameraImages(loc_request_data);
sentrequestCount++;
}
}
else
{
CancelInvoke("RequestLocalization");
}
}
/// <summary>
/// The ARCameraManager is needed to acquire the frame to send to scapekit's backend
/// </summary>
/// <param name="arCameraManager">
/// The arCameraManager usually attached to the ARCamera
/// </param>
public virtual void SetCameraManager(ARCameraManager arCameraManager)
{
this.scapeSessionNative.SetCameraManager(arCameraManager);
}
