private IEnumerator EmitFrame()
{
frameCounter += 1;
// we should only read the screen buffer after rendering is complete
yield return(new WaitForEndOfFrame());
WWWForm form = new WWWForm();
MultiAgentMetadata multiMeta = new MultiAgentMetadata();
multiMeta.agents = new MetadataWrapper[this.agents.Count];
ThirdPartyCameraMetadata[] cameraMetadata = new ThirdPartyCameraMetadata[this.thirdPartyCameras.Count];
multiMeta.activeAgentId = this.activeAgentId;
multiMeta.sequenceId = this.currentSequenceId;
RenderTexture currentTexture = RenderTexture.active;
for (int i = 0; i < this.thirdPartyCameras.Count; i++)
{
ThirdPartyCameraMetadata cMetadata = new ThirdPartyCameraMetadata();
Camera camera = thirdPartyCameras.ToArray()[i];
cMetadata.thirdPartyCameraId = i;
cMetadata.position = camera.gameObject.transform.position;
cMetadata.rotation = camera.gameObject.transform.eulerAngles;
cameraMetadata[i] = cMetadata;
ImageSynthesis imageSynthesis = camera.gameObject.GetComponentInChildren <ImageSynthesis> () as ImageSynthesis;
addThirdPartyCameraImageForm(form, camera);
addImageSynthesisImageForm(form, imageSynthesis, this.renderDepthImage, "_depth", "image_thirdParty_depth");
addImageSynthesisImageForm(form, imageSynthesis, this.renderNormalsImage, "_normals", "image_thirdParty_normals");
addImageSynthesisImageForm(form, imageSynthesis, this.renderObjectImage, "_id", "image_thirdParty_image_ids");
addImageSynthesisImageForm(form, imageSynthesis, this.renderClassImage, "_class", "image_thirdParty_classes");
}
for (int i = 0; i < this.agents.Count; i++)
{
BaseFPSAgentController agent = this.agents.ToArray() [i];
if (i > 0)
{
this.agents.ToArray() [i - 1].m_Camera.enabled = false;
}
agent.m_Camera.enabled = true;
MetadataWrapper metadata = agent.generateMetadataWrapper();
metadata.agentId = i;
// we don't need to render the agent's camera for the first agent
addImageForm(form, agent);
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderDepthImage, "_depth", "image_depth");
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderNormalsImage, "_normals", "image_normals");
addObjectImageForm(form, agent, ref metadata);
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderClassImage, "_class", "image_classes");
metadata.thirdPartyCameras = cameraMetadata;
multiMeta.agents [i] = metadata;
}
if (this.agents.Count != 1)
{
this.agents.ToArray()[this.agents.Count - 1].m_Camera.enabled = false;
}
this.agents.ToArray()[0].m_Camera.enabled = true;
RenderTexture.active = currentTexture;
//form.AddField("metadata", JsonUtility.ToJson(multiMeta));
form.AddField("metadata", Newtonsoft.Json.JsonConvert.SerializeObject(multiMeta));
form.AddField("token", robosimsClientToken);
#if !UNITY_WEBGL
using (var www = UnityWebRequest.Post("http://" + robosimsHost + ":" + robosimsPort + "/train", form))
{
yield return(www.SendWebRequest());
if (www.isNetworkError || www.isHttpError)
{
Debug.Log("Error: " + www.error);
yield break;
}
ProcessControlCommand(www.downloadHandler.text);
}
#endif
}
public IEnumerator EmitFrame()
{
frameCounter += 1;
bool shouldRender = this.renderImage && serverSideScreenshot;
if (shouldRender)
{
// we should only read the screen buffer after rendering is complete
yield return(new WaitForEndOfFrame());
}
WWWForm form = new WWWForm();
MultiAgentMetadata multiMeta = new MultiAgentMetadata();
multiMeta.agents = new MetadataWrapper[this.agents.Count];
multiMeta.activeAgentId = this.activeAgentId;
multiMeta.sequenceId = this.currentSequenceId;
ThirdPartyCameraMetadata[] cameraMetadata = new ThirdPartyCameraMetadata[this.thirdPartyCameras.Count];
RenderTexture currentTexture = null;
JavaScriptInterface jsInterface = null;
if (shouldRender)
{
currentTexture = RenderTexture.active;
for (int i = 0; i < this.thirdPartyCameras.Count; i++)
{
ThirdPartyCameraMetadata cMetadata = new ThirdPartyCameraMetadata();
Camera camera = thirdPartyCameras.ToArray()[i];
cMetadata.thirdPartyCameraId = i;
cMetadata.position = camera.gameObject.transform.position;
cMetadata.rotation = camera.gameObject.transform.eulerAngles;
cameraMetadata[i] = cMetadata;
ImageSynthesis imageSynthesis = camera.gameObject.GetComponentInChildren <ImageSynthesis> () as ImageSynthesis;
addThirdPartyCameraImageForm(form, camera);
addImageSynthesisImageForm(form, imageSynthesis, this.renderDepthImage, "_depth", "image_thirdParty_depth");
addImageSynthesisImageForm(form, imageSynthesis, this.renderNormalsImage, "_normals", "image_thirdParty_normals");
addImageSynthesisImageForm(form, imageSynthesis, this.renderObjectImage, "_id", "image_thirdParty_image_ids");
addImageSynthesisImageForm(form, imageSynthesis, this.renderClassImage, "_class", "image_thirdParty_classes");
addImageSynthesisImageForm(form, imageSynthesis, this.renderClassImage, "_flow", "image_thirdParty_flow");//XXX fix this in a bit
}
}
for (int i = 0; i < this.agents.Count; i++)
{
BaseFPSAgentController agent = this.agents.ToArray() [i];
jsInterface = agent.GetComponent <JavaScriptInterface>();
MetadataWrapper metadata = agent.generateMetadataWrapper();
metadata.agentId = i;
// we don't need to render the agent's camera for the first agent
if (shouldRender)
{
addImageForm(form, agent);
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderDepthImage, "_depth", "image_depth");
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderNormalsImage, "_normals", "image_normals");
addObjectImageForm(form, agent, ref metadata);
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderClassImage, "_class", "image_classes");
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderFlowImage, "_flow", "image_flow");
metadata.thirdPartyCameras = cameraMetadata;
}
multiMeta.agents [i] = metadata;
}
if (shouldRender)
{
RenderTexture.active = currentTexture;
}
var serializedMetadata = Newtonsoft.Json.JsonConvert.SerializeObject(multiMeta);
#if UNITY_WEBGL
// JavaScriptInterface jsI = FindObjectOfType<JavaScriptInterface>();
// jsInterface.SendAction(new ServerAction(){action = "Test"});
if (jsInterface != null)
{
jsInterface.SendActionMetadata(serializedMetadata);
}
#endif
//form.AddField("metadata", JsonUtility.ToJson(multiMeta));
form.AddField("metadata", serializedMetadata);
form.AddField("token", robosimsClientToken);
#if !UNITY_WEBGL
if (synchronousHttp)
{
if (this.sock == null)
{
// Debug.Log("connecting to host: " + robosimsHost);
IPAddress host = IPAddress.Parse(robosimsHost);
IPEndPoint hostep = new IPEndPoint(host, robosimsPort);
this.sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
try {
this.sock.Connect(hostep);
}
catch (SocketException e) {
Debug.Log("Socket exception: " + e.ToString());
}
}
if (this.sock != null && this.sock.Connected)
{
byte[] rawData = form.data;
string request = "POST /train HTTP/1.1\r\n" +
"Content-Length: " + rawData.Length.ToString() + "\r\n";
foreach (KeyValuePair <string, string> entry in form.headers)
{
request += entry.Key + ": " + entry.Value + "\r\n";
}
request += "\r\n";
int sent = this.sock.Send(Encoding.ASCII.GetBytes(request));
sent = this.sock.Send(rawData);
// waiting for a frame here keeps the Unity window in sync visually
// its not strictly necessary, but allows the interact() command to work properly
// and does not reduce the overall FPS
yield return(new WaitForEndOfFrame());
byte[] headerBuffer = new byte[1024];
int bytesReceived = 0;
byte[] bodyBuffer = null;
int bodyBytesReceived = 0;
int contentLength = 0;
// read header
while (true)
{
int received = this.sock.Receive(headerBuffer, bytesReceived, headerBuffer.Length - bytesReceived, SocketFlags.None);
if (received == 0)
{
Debug.LogError("0 bytes received attempting to read header - connection closed");
break;
}
bytesReceived += received;;
string headerMsg = Encoding.ASCII.GetString(headerBuffer, 0, bytesReceived);
int offset = headerMsg.IndexOf("\r\n\r\n");
if (offset > 0)
{
contentLength = parseContentLength(headerMsg.Substring(0, offset));
bodyBuffer = new byte[contentLength];
bodyBytesReceived = bytesReceived - (offset + 4);
Array.Copy(headerBuffer, offset + 4, bodyBuffer, 0, bodyBytesReceived);
break;
}
}
// read body
while (bodyBytesReceived < contentLength)
{
// check for 0 bytes received
int received = this.sock.Receive(bodyBuffer, bodyBytesReceived, bodyBuffer.Length - bodyBytesReceived, SocketFlags.None);
if (received == 0)
{
Debug.LogError("0 bytes received attempting to read body - connection closed");
break;
}
bodyBytesReceived += received;
//Debug.Log("total bytes received: " + bodyBytesReceived);
}
string msg = Encoding.ASCII.GetString(bodyBuffer, 0, bodyBytesReceived);
ProcessControlCommand(msg);
}
}
else
{
using (var www = UnityWebRequest.Post("http://" + robosimsHost + ":" + robosimsPort + "/train", form))
{
yield return(www.SendWebRequest());
if (www.isNetworkError || www.isHttpError)
{
Debug.Log("Error: " + www.error);
yield break;
}
ProcessControlCommand(www.downloadHandler.text);
}
}
#endif
}
private IEnumerator EmitFrame()
{
frameCounter += 1;
bool shouldRender = this.renderImage && serverSideScreenshot;
if (shouldRender)
{
// we should only read the screen buffer after rendering is complete
yield return(new WaitForEndOfFrame());
if (synchronousHttp)
{
// must wait an additional frame when in synchronous mode otherwise the frame lags
yield return(new WaitForEndOfFrame());
}
}
WWWForm form = new WWWForm();
MultiAgentMetadata multiMeta = new MultiAgentMetadata();
multiMeta.agents = new MetadataWrapper[this.agents.Count];
multiMeta.activeAgentId = this.activeAgentId;
multiMeta.sequenceId = this.currentSequenceId;
ThirdPartyCameraMetadata[] cameraMetadata = new ThirdPartyCameraMetadata[this.thirdPartyCameras.Count];
RenderTexture currentTexture = null;
if (shouldRender)
{
currentTexture = RenderTexture.active;
for (int i = 0; i < this.thirdPartyCameras.Count; i++)
{
ThirdPartyCameraMetadata cMetadata = new ThirdPartyCameraMetadata();
Camera camera = thirdPartyCameras.ToArray()[i];
cMetadata.thirdPartyCameraId = i;
cMetadata.position = camera.gameObject.transform.position;
cMetadata.rotation = camera.gameObject.transform.eulerAngles;
cameraMetadata[i] = cMetadata;
ImageSynthesis imageSynthesis = camera.gameObject.GetComponentInChildren <ImageSynthesis> () as ImageSynthesis;
addThirdPartyCameraImageForm(form, camera);
addImageSynthesisImageForm(form, imageSynthesis, this.renderDepthImage, "_depth", "image_thirdParty_depth");
addImageSynthesisImageForm(form, imageSynthesis, this.renderNormalsImage, "_normals", "image_thirdParty_normals");
addImageSynthesisImageForm(form, imageSynthesis, this.renderObjectImage, "_id", "image_thirdParty_image_ids");
addImageSynthesisImageForm(form, imageSynthesis, this.renderClassImage, "_class", "image_thirdParty_classes");
}
}
for (int i = 0; i < this.agents.Count; i++)
{
BaseFPSAgentController agent = this.agents.ToArray() [i];
MetadataWrapper metadata = agent.generateMetadataWrapper();
metadata.agentId = i;
// we don't need to render the agent's camera for the first agent
if (shouldRender)
{
addImageForm(form, agent);
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderDepthImage, "_depth", "image_depth");
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderNormalsImage, "_normals", "image_normals");
addObjectImageForm(form, agent, ref metadata);
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderClassImage, "_class", "image_classes");
metadata.thirdPartyCameras = cameraMetadata;
}
multiMeta.agents [i] = metadata;
}
if (shouldRender)
{
RenderTexture.active = currentTexture;
}
//form.AddField("metadata", JsonUtility.ToJson(multiMeta));
form.AddField("metadata", Newtonsoft.Json.JsonConvert.SerializeObject(multiMeta));
form.AddField("token", robosimsClientToken);
#if !UNITY_WEBGL
if (synchronousHttp)
{
if (this.sock == null)
{
// Debug.Log("connecting to host: " + robosimsHost);
IPAddress host = IPAddress.Parse(robosimsHost);
IPEndPoint hostep = new IPEndPoint(host, robosimsPort);
try
{
Socket s = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
s.Connect(hostep);
this.sock = s;
}
catch (SocketException ex)
{
# if UNITY_EDITOR
// swallowing the error since its fine to run without the Python side with the editor
yield break;
# else
throw ex;
# endif
}
}
