/// <summary>)
/// Serialize the image that was taken and turn it into a rawrequest.
/// 1. Take photo and decode it as jpeg string string
/// 2. decode the jpeg wtih base64 to be serializeable
/// 3. serialize everything as json string
/// 4. serialize the json string as raw request
/// </summary>
private IEnumerator <bool> SerializeRequest(PhotoCaptureFrame photoCapturedFrame)
{
yield return(true);
//Texture2D tex = new Texture2D(ImageCapture.Instance.width,
// ImageCapture.Instance.height);
//photoCapturedFrame.UploadImageDataToTexture(tex);
//byte[] jpgEncoded = tex.EncodeToJPG
List <byte> jpgEncodedList = new List <byte>();
photoCapturedFrame.CopyRawImageDataIntoBuffer(jpgEncodedList);
byte[] jpgEncoded = jpgEncodedList.ToArray();
// server expects an base64 encoded JPG encoded string
// should have the form {"inputs": [{"b64": <b64encodejpgencodedstring>}]}
string b64Encode = Convert.ToBase64String(jpgEncoded);
DetectionRequest detectionRequest = new DetectionRequest {
inputs = new List <B64> {
new B64 {
b64 = b64Encode
}
}
};
string jsonRequest = JsonConvert.SerializeObject(detectionRequest);
RequestBufferElem requestBufferElem = new RequestBufferElem()
{
rawRequest = Encoding.UTF8.GetBytes(jsonRequest)
};
if (!photoCapturedFrame.TryGetCameraToWorldMatrix(out requestBufferElem.cameraToWorld) ||
!photoCapturedFrame.TryGetProjectionMatrix(out requestBufferElem.projection))
{
requestBufferElem.hasWorldData = false;
}
else
{
requestBufferElem.hasWorldData = true;
}
photoCapturedFrame.Dispose();
rawRequestBuffer.Enqueue(requestBufferElem);
rawRequestBufferEmpty = false;
timestamp = stopwatch.ElapsedMilliseconds;
}
void Update()
{
if (outputsBufferRefill && !ImageSerializer.Instance.rawRequestBufferEmpty && stopwatch.ElapsedMilliseconds - timestamp > 200)
{
//For Timing Analysis
UnityEngine.Debug.Log($"ImageAnalysis Idletime {stopwatch.ElapsedMilliseconds - timestamp} Runtime {timestamp}");
stopwatch = Stopwatch.StartNew();
//Deqeue a rawrequest element from the buffer for the Worker
RequestBufferElem rawRequestElem = ImageSerializer.Instance.rawRequestBuffer.Dequeue();
ImageSerializer.Instance.rawRequestBufferRefill = true;
ImageSerializer.Instance.rawRequestBufferEmpty = true;
outputsBufferRefill = false;
//Start Analysis Worker
StartCoroutine(AnalyseLastImageCaptured(rawRequestElem));
}
}
/// <summary>
/// Call the Computer Vision Service to submit the image.
/// </summary>
public IEnumerator AnalyseLastImageCaptured(RequestBufferElem rawRequestElem)
{
// define header as json
//Dictionary<string, string> headers = new Dictionary<string, string>();
//headers.Add("Content-Type", "application/json");
DetectionResponse detectionResponse = new DetectionResponse();
//send of request
//using (WWW www = new WWW(predictionEndpoint, rawRequest, headers))
//{
// yield return www;
// //Debug.Log($"Response Error {www.error}");
// //Debug.Log(www.text);
// detectionResponse = JsonConvert.DeserializeObject<DetectionResponse>(www.text);
//}
WWWForm webForm = new WWWForm();
using (UnityWebRequest unityWebRequest = UnityWebRequest.Post(predictionEndpoint, webForm))
{
//Set headers
unityWebRequest.SetRequestHeader("Content-Type", "application/json");
//Upload Handler to handle data
unityWebRequest.uploadHandler = new UploadHandlerRaw(rawRequestElem.rawRequest);
unityWebRequest.uploadHandler.contentType = "application/json";
// The download handler will help receiving the analysis from Azure
unityWebRequest.downloadHandler = new DownloadHandlerBuffer();
// Send the request
yield return(unityWebRequest.SendWebRequest());
//Decode the json response
detectionResponse = JsonConvert.DeserializeObject <DetectionResponse>(unityWebRequest.downloadHandler.text);
}
//check if the response was succesful
if (detectionResponse.outputs != null)
{
//// Uncomment this to debug the values in the response
//detectionResponse.outputs = ValuesForCalibration(detectionResponse.outputs);
//// uncomment this to debug the values in the response
//DebugLogResponse(detectionResponse.outputs);
//If world data is available get it
if (rawRequestElem.hasWorldData)
{
detectionResponse.outputs.hasWorldData = true;
detectionResponse.outputs.cameraToWorld = rawRequestElem.cameraToWorld;
detectionResponse.outputs.projection = rawRequestElem.projection;
}
else
{
detectionResponse.outputs.hasWorldData = false;
}
//enquue the response and quit worker
outputsBuffer.Enqueue(detectionResponse.outputs);
outputsBufferEmpty = false;
//For Timining Analysis
timestamp = stopwatch.ElapsedMilliseconds;
}
else
{
// The Server failed, Show with cursor
SceneOrganizer.Instance.cursor.GetComponent <Renderer>().material.color = Color.blue;
// Write a debug message
UnityEngine.Debug.Log("The Server failed");
// Hibernate the detection for 2sec
timestamp = stopwatch.ElapsedMilliseconds + 1800;
}
}
