private void GetImageAsync()
{
currentXRImage = new XRCameraImage();
if (cameraManager.TryGetLatestImage(out currentXRImage))
{
currentXRImage.ConvertAsync(new XRCameraImageConversionParams
{
inputRect = new RectInt(0, 0, currentXRImage.width, currentXRImage.height),
outputDimensions = new Vector2Int(ScapeImgWidth, ScapeImgHeight),
outputFormat = TextureFormat.R8
}, ProcessImage);
currentXRImage.Dispose();
ScapeLogging.LogDebug("GetImageAsync() " + (Time.time - requestTime));
measurementsRequested = false;
}
}
// Convert image to correct size, rotate it, send it off to TorchServe, and draw resulting bounding boxes
private IEnumerator ProcessImage(XRCameraImage image)
{
using (var request = image.ConvertAsync(new XRCameraImageConversionParams
{
inputRect = new RectInt(0, 0, image.width, image.height),
outputDimensions = new Vector2Int((int)Width, (int)Height),
outputFormat = TextureFormat.RGBA32,
transformation = CameraImageTransformation.MirrorY
}))
{
while (!request.status.IsDone())
{
yield return(null);
}
if (request.status != AsyncCameraImageConversionStatus.Ready)
{
Debug.LogErrorFormat("Image request failed with status {0}", request.status);
yield break;
}
_currentJpgBytes = ConvertBufferToJpg(request.GetData <byte>(), request.conversionParams);
}
using (var client = new HttpClient {
Timeout = TimeSpan.FromMilliseconds(2000)
})
{
if (settingsManager.SettingsModel.activeEndpoint?.url == null)
{
Debug.LogWarning("Null Model Endpoint URL");
yield break;
}
var content = new ByteArrayContent(_currentJpgBytes);
content.Headers.Add("Content-Type", "image/jpg");
var startTime = Time.realtimeSinceStartup;
Task <HttpResponseMessage> webRequestTask;
try
{
_activeRequests += 1;
webRequestTask = client.PostAsync(new Uri(settingsManager.SettingsModel.activeEndpoint?.url), content);
}
catch (Exception e)
{
_activeRequests -= 1;
Console.WriteLine(e);
yield break;
}
yield return(Utils.WaitForTaskToComplete(webRequestTask));
_activeRequests -= 1;
Console.WriteLine("Round trip: " + (Time.realtimeSinceStartup - startTime));
if (!webRequestTask.Result.IsSuccessStatusCode)
{
Debug.LogErrorFormat("Error While Sending: {0}", webRequestTask.Result.ReasonPhrase);
yield break;
}
var stringReadingTask = webRequestTask.Result.Content.ReadAsStringAsync();
yield return(Utils.WaitForTaskToComplete(stringReadingTask));
// Wrap output in top-level object for JsonUtility
var text = "{\"objects\":" + stringReadingTask.Result + "}";
// If JSON output does not have an array, the response was not a 200 OK
if (!text.Contains("["))
{
Debug.LogErrorFormat("Prediction error: {0}\n", stringReadingTask.Result);
yield break;
}
var rotation = RotationForScreenOrientation();
if (!rotation.HasValue)
{
Debug.LogErrorFormat("Invalid screen orientation: {0}", _orientationObserver.ScreenOrientation);
yield break;
}
_currentClassifications = JsonUtility.FromJson <JsonWrapper>(text).objects
.FindAll(it => it.score >= settingsManager.SettingsModel.predictionScoreThreshold);
var classifications = _currentClassifications
.ConvertAll(old => new ObjectClassification(old.label,
old.label_id,
old.box
.RotatedBy(rotation.Value, new Vector2(Width, Height))
.ScaledBy(ScaleFactor),
old.score));
boundingBoxManager.SetObjectClassifications(classifications);
}
}
IEnumerator ProcessImage(XRCameraImage image)
{
// Create the async conversion request.
var request = image.ConvertAsync(new XRCameraImageConversionParams
{
// Use the full image.
inputRect = new RectInt(0, 0, image.width, image.height),
// Downsample by 2.
outputDimensions = new Vector2Int(image.width / 2, image.height / 2),
// Color image format.
outputFormat = TextureFormat.RGB24,
// Flip across the Y axis.
transformation = CameraImageTransformation.MirrorY
});
// Wait for the conversion to complete.
while (!request.status.IsDone())
{
yield return(null);
}
// Check status to see if the conversion completed successfully.
if (request.status != AsyncCameraImageConversionStatus.Ready)
{
// Something went wrong.
Debug.LogErrorFormat("Request failed with status {0}", request.status);
// Dispose even if there is an error.
request.Dispose();
yield break;
}
// Image data is ready. Let's apply it to a Texture2D.
var rawData = request.GetData <byte>();
m_LastCameraTexture = new Texture2D(
request.conversionParams.outputDimensions.x,
request.conversionParams.outputDimensions.y,
request.conversionParams.outputFormat,
false);
// Copy the image data into the texture.
m_LastCameraTexture.LoadRawTextureData(rawData);
m_LastCameraTexture.Apply();
// Need to dispose the request to delete resources associated
// with the request, including the raw data.
request.Dispose();
// enable mesh + set shader properties
arInstance.gameObject.SetActive(true);
arInstance.transform.position = ARCamera.transform.position;
arInstance.transform.rotation = ARCamera.transform.rotation;
mat.SetTexture("Albedo", m_LastCameraTexture);
// update ui
raw.texture = m_LastCameraTexture;
// TODO update size of image to match texture
}
IEnumerator ProcessImage(XRCameraImage image)
{
// Create the async conversion request
var request = image.ConvertAsync(new XRCameraImageConversionParams
{
// Use the full image
inputRect = new RectInt(0, 0, image.width, image.height),
// Downsample by 2
outputDimensions = new Vector2Int(image.width / 8, image.height / 8),
// Color image format
outputFormat = TextureFormat.RGB24,
// Flip across the Y axis
transformation = CameraImageTransformation.MirrorY
});
// Wait for it to complete
while (!request.status.IsDone())
{
yield return(null);
}
// Check status to see if it completed successfully.
if (request.status != AsyncCameraImageConversionStatus.Ready)
{
// Something went wrong
Debug.LogErrorFormat("Request failed with status {0}", request.status);
// Dispose even if there is an error.
request.Dispose();
yield break;
}
// Image data is ready. Let's apply it to a Texture2D.
var rawData = request.GetData <byte>();
Texture2D texture = new Texture2D(
request.conversionParams.outputDimensions.x,
request.conversionParams.outputDimensions.y,
request.conversionParams.outputFormat,
false);
// Copy the image data into the texture
texture.LoadRawTextureData(rawData);
texture.Apply();
texture.Compress(false);
m_Texture = Common.Util.TextureUtil.AnticlockwiseRotate90(texture);
Debug.Log("生成图片！");
byte[] bytes = m_Texture.EncodeToPNG();
//SaveImage("temp", bytes);
ImageRecognition(bytes);
// Need to dispose the request to delete resources associated
// with the request, including the raw data.
request.Dispose();
}
