private Vector3 agentStartPosition(BaseFPSAgentController agent)
{
Transform t = agent.transform;
Vector3[] castDirections = new Vector3[] { t.forward, t.forward * -1, t.right, t.right * -1 };
RaycastHit maxHit = new RaycastHit();
Vector3 maxDirection = Vector3.zero;
RaycastHit hit;
CharacterController charContr = agent.m_CharacterController;
Vector3 p1 = t.position + charContr.center + Vector3.up * -charContr.height * 0.5f;
Vector3 p2 = p1 + Vector3.up * charContr.height;
foreach (Vector3 d in castDirections)
{
if (Physics.CapsuleCast(p1, p2, charContr.radius, d, out hit))
{
if (hit.distance > maxHit.distance)
{
maxHit = hit;
maxDirection = d;
}
}
}
if (maxHit.distance > (charContr.radius * 5))
{
return(t.position + (maxDirection * (charContr.radius * 4)));
}
return(Vector3.zero);
}
//returns grid points where there is an experiment receptacle (screens too) on the table
//this only returns areas where the ReceptacleTriggerBox of the object is, not the geometry of the object itself
//use agent's current forward as directionality- agent forward and agent left
//the y value will be wherever the agent's hand currently is
//gridIncrement- size between grid points in meters
//count - casts a grid forward <2 * count + 1> by <count>
public List <Vector3> ValidGrid(Vector3 origin, float gridIncrement, int count, BaseFPSAgentController agent)
{
//start from origin which will be agent's hand
List <Vector3> pointsOnGrid = new List <Vector3>();
for (int i = 0; i < count; i++)
{
//from origin, go gridIncrement a number of times equal to gridDimension
//in the agent's forward direction
Vector3 thisPoint = origin + agent.transform.forward * gridIncrement * i;
pointsOnGrid.Add(thisPoint);
//then, from this point, go gridDimension times in both left and right direction
for (int j = 1; j < count + 1; j++)
{
pointsOnGrid.Add(thisPoint + agent.transform.right * gridIncrement * j);
pointsOnGrid.Add(thisPoint + -agent.transform.right * gridIncrement * j);
}
}
// #if UNITY_EDITOR
// debugCoords = pointsOnGrid;
// #endif
List <Vector3> actualPoints = new List <Vector3>();
RaycastHit[] hits;
foreach (Vector3 point in pointsOnGrid)
{
hits = Physics.RaycastAll(point + new Vector3(0, 5, 0), Vector3.down, 20.0f, 1 << 9, QueryTriggerInteraction.Collide);
float[] hitDistances = new float[hits.Length];
for (int i = 0; i < hitDistances.Length; i++)
{
hitDistances[i] = hits[i].distance;
}
Array.Sort(hitDistances, hits);
foreach (RaycastHit h in hits)
{
if (h.transform.GetComponent <SimObjPhysics>())
{
var o = h.transform.GetComponent <SimObjPhysics>();
if (o.Type != SimObjType.DiningTable && o.Type != SimObjType.Floor)
{
actualPoints.Add(point);
}
}
}
}
// #if UNITY_EDITOR
// debugCoords = actualPoints;
// #endif
//ok we now have grid points in a grid, now raycast down from each of those and see if we hit a receptacle...
return(actualPoints);
}
public void Initialize(ServerAction action)
{
if (action.agentType != null && action.agentType.ToLower() == "stochastic")
{
this.agents.Clear();
// stochastic must have these set to work properly
action.continuous = true;
action.snapToGrid = false;
GameObject fpsController = GameObject.FindObjectOfType <BaseFPSAgentController>().gameObject;
primaryAgent.enabled = false;
primaryAgent = fpsController.GetComponent <StochasticRemoteFPSAgentController>();
primaryAgent.agentManager = this;
primaryAgent.enabled = true;
// must manually call start here since it this only gets called before Update() is called
primaryAgent.Start();
this.agents.Add(primaryAgent);
}
primaryAgent.ProcessControlCommand(action);
primaryAgent.IsVisible = action.makeAgentsVisible;
this.renderClassImage = action.renderClassImage;
this.renderDepthImage = action.renderDepthImage;
this.renderNormalsImage = action.renderNormalsImage;
this.renderObjectImage = action.renderObjectImage;
this.renderFlowImage = action.renderFlowImage;
if (action.alwaysReturnVisibleRange)
{
((PhysicsRemoteFPSAgentController)primaryAgent).alwaysReturnVisibleRange = action.alwaysReturnVisibleRange;
}
StartCoroutine(addAgents(action));
}
public void GatherSimObjPhysInScene()
{
List <SimObjPhysics> allPhysObjects = new List <SimObjPhysics>();
allPhysObjects.AddRange(FindObjectsOfType <SimObjPhysics>());
allPhysObjects.Sort((x, y) => (x.Type.ToString().CompareTo(y.Type.ToString())));
foreach (SimObjPhysics o in allPhysObjects)
{
Generate_ObjectID(o);
///debug in editor, make sure no two object share ids for some reason
#if UNITY_EDITOR
if (CheckForDuplicateObjectIDs(o))
{
Debug.Log("Yo there are duplicate ObjectIDs! Check" + o.ObjectID + "in scene " + UnityEngine.SceneManagement.SceneManager.GetActiveScene().name);
}
else
{
AddToObjectsInScene(o);
continue;
}
#endif
AddToObjectsInScene(o);
}
BaseFPSAgentController fpsController = GameObject.FindObjectOfType <BaseFPSAgentController>();
if (fpsController.imageSynthesis != null)
{
fpsController.imageSynthesis.OnSceneChange();
}
}
private void initializePrimaryAgent()
{
GameObject fpsController = GameObject.FindObjectOfType <BaseFPSAgentController>().gameObject;
primaryAgent = fpsController.GetComponent <PhysicsRemoteFPSAgentController>();
primaryAgent.enabled = true;
primaryAgent.agentManager = this;
primaryAgent.actionComplete = true;
}
private void initializePrimaryAgent()
{
GameObject fpsController = GameObject.Find("FPSController");
PhysicsRemoteFPSAgentController physicsAgent = fpsController.GetComponent <PhysicsRemoteFPSAgentController>();
primaryAgent = physicsAgent;
primaryAgent.agentManager = this;
primaryAgent.enabled = true;
primaryAgent.actionComplete = true;
}
private void updateAgentColor(BaseFPSAgentController agent, Color color)
{
foreach (MeshRenderer r in agent.gameObject.GetComponentsInChildren <MeshRenderer> () as MeshRenderer[])
{
foreach (Material m in r.materials)
{
m.color = color;
}
}
}
public IEnumerator TestRotateRight()
{
yield return(ExecuteDebugAction("init"));
BaseFPSAgentController agent = GameObject.FindObjectOfType <BaseFPSAgentController>();
agent.gameObject.transform.rotation = Quaternion.Euler(Vector3.zero);
yield return(ExecuteDebugAction("rr"));
Assert.AreEqual((int)agent.gameObject.transform.rotation.eulerAngles.y, 90);
}
private void addImageForm(WWWForm form, BaseFPSAgentController agent)
{
if (this.renderImage)
{
if (this.agents.Count > 1 || this.thirdPartyCameras.Count > 0)
{
RenderTexture.active = agent.m_Camera.activeTexture;
agent.m_Camera.Render();
}
form.AddBinaryData("image", captureScreen());
}
}
//same as PlaceObjectReceptacle but instead only succeeds if final placed object is within viewport
public bool PlaceObjectReceptacleInViewport(List <ReceptacleSpawnPoint> rsps, SimObjPhysics sop, bool PlaceStationary, int maxPlacementAttempts, int degreeIncrement, bool AlwaysPlaceUpright)
{
if (rsps == null)
{
#if UNITY_EDITOR
Debug.Log("Null list of points to check, please pass in populated list of <ReceptacleSpawnPoint>?");
#endif
return(false); //uh, there was nothing in the List for some reason, so failed to spawn
}
if (rsps.Count == 0)
{
return(false);
}
List <ReceptacleSpawnPoint> goodRsps = new List <ReceptacleSpawnPoint>();
foreach (ReceptacleSpawnPoint p in rsps)
{
if (!p.ParentSimObjPhys.GetComponent <SimObjPhysics>().DoesThisObjectHaveThisSecondaryProperty
(SimObjSecondaryProperty.ObjectSpecificReceptacle))
{
goodRsps.Add(p);
}
}
int tries = 0;
foreach (ReceptacleSpawnPoint p in goodRsps)
{
//if this is an Object Specific Receptacle, stop this check right now! I mean it!
//Placing objects in/on an Object Specific Receptacle uses different logic to place the
//object at the Attachemnet point rather than in the spawn area, so stop this right now!
if (PlaceObject(sop, p, PlaceStationary, degreeIncrement, AlwaysPlaceUpright))
{
//check to make sure the placed object is within the viewport
BaseFPSAgentController primaryAgent = GameObject.Find("PhysicsSceneManager").GetComponent <AgentManager>().ReturnPrimaryAgent();
if (primaryAgent.GetComponent <PhysicsRemoteFPSAgentController>().objectIsOnScreen(sop))
{
return(true);
}
}
tries += 1;
if (maxPlacementAttempts > 0 && tries > maxPlacementAttempts)
{
break;
}
}
//couldn't find valid places to spawn
return(false);
}
public void UpdateAgentColor(BaseFPSAgentController agent, Color color)
{
foreach (MeshRenderer r in agent.gameObject.GetComponentsInChildren <MeshRenderer> () as MeshRenderer[])
{
foreach (Material m in r.materials)
{
if (m.name.Contains("Agent_Color_Mat"))
{
m.color = color;
}
}
}
}
private void addDepthImageForm(WWWForm form, BaseFPSAgentController agent)
{
if (this.renderDepthImage)
{
if (!agent.imageSynthesis.hasCapturePass("_depth"))
{
Debug.LogError("Depth image not available - returning empty image");
}
byte[] bytes = agent.imageSynthesis.Encode("_depth");
form.AddBinaryData("image_depth", bytes);
}
}
private void addAgent(ServerAction action)
{
Vector3 clonePosition = new Vector3(action.x, action.y, action.z);
BaseFPSAgentController clone = UnityEngine.Object.Instantiate(primaryAgent);
clone.IsVisible = action.makeAgentsVisible;
clone.actionDuration = this.actionDuration;
// clone.m_Camera.targetDisplay = this.agents.Count;
clone.transform.position = clonePosition;
UpdateAgentColor(clone, agentColors[this.agents.Count]);
clone.ProcessControlCommand(action);
this.agents.Add(clone);
}
public IEnumerator PlaceHeldObject_Deprecated_Z_XY_PutNearXY_True()
{
Dictionary <string, object> action = new Dictionary <string, object>();
action["action"] = "Initialize";
action["fieldOfView"] = 90f;
action["snapToGrid"] = true;
yield return(ExecuteAction(action));
action.Clear();
action["action"] = "RotateRight";
yield return(ExecuteAction(action));
yield return(ExecuteAction(action));
action.Clear();
action["action"] = "LookDown";
yield return(ExecuteAction(action));
action.Clear();
action["action"] = "MoveRight";
yield return(ExecuteAction(action));
action.Clear();
action["action"] = "PickupObject";
action["objectId"] = "CreditCard|-00.46|+01.10|+00.87";
yield return(ExecuteAction(action));
action.Clear();
action["action"] = "PlaceHeldObject";
action["x"] = 0.5f;
action["y"] = 0.5f;
action["putNearXY"] = true;
// this should cause the exception
action["z"] = 5.0f;
yield return(ExecuteAction(action));
BaseFPSAgentController agent = GameObject.FindObjectOfType <BaseFPSAgentController>();
Assert.AreEqual(agent.lastActionSuccess, false);
}
private void addAgent(ServerAction action)
{
Vector3 clonePosition = new Vector3(action.x, action.y, action.z);
//disable ambient occlusion on primary agetn because it causes issues with multiple main cameras
//primaryAgent.GetComponent<PhysicsRemoteFPSAgentController>().DisableScreenSpaceAmbientOcclusion();
BaseFPSAgentController clone = UnityEngine.Object.Instantiate(primaryAgent);
clone.IsVisible = action.makeAgentsVisible;
clone.actionDuration = this.actionDuration;
// clone.m_Camera.targetDisplay = this.agents.Count;
clone.transform.position = clonePosition;
UpdateAgentColor(clone, agentColors[this.agents.Count]);
clone.ProcessControlCommand(action);
this.agents.Add(clone);
}
// action to be called from PhysicsRemoteFPSAgentController
public void Slice()
{
// if this is already sliced, we can't slice again so yeah stop that
if (isSliced == true)
{
return;
}
// Disable this game object and spawn in the broken pieces
Rigidbody rb = gameObject.GetComponent <Rigidbody>();
rb.collisionDetectionMode = CollisionDetectionMode.Discrete;
rb.isKinematic = true;
// turn off everything except the top object, so we can continue to report back isSliced meta info without the object being "active"
foreach (Transform t in gameObject.transform)
{
t.gameObject.SetActive(false);
}
GameObject resultObject;
if (!gameObject.GetComponent <SimObjPhysics>().DoesThisObjectHaveThisSecondaryProperty(SimObjSecondaryProperty.CanBeCooked))
{
// instantiate the normal object if this object is not cooked, otherwise....
resultObject = Instantiate(ObjectToChangeTo, transform.position, transform.rotation);
isSliced = true;
}
// if the object can be cooked, check if it is cooked and then spawn the cooked object to change to, otherwise spawn the normal object
else
{
// instantiate the normal object if this object is not cooked, otherwise....
resultObject = Instantiate(ObjectToChangeTo, transform.position, transform.rotation);
isSliced = true;
if (gameObject.GetComponent <CookObject>().IsCooked())
{
// cook all objects under the resultObject
foreach (Transform t in resultObject.transform)
{
t.GetComponent <CookObject>().Cook();
}
}
}
PhysicsSceneManager psm = GameObject.Find("PhysicsSceneManager").GetComponent <PhysicsSceneManager>();
if (psm != null)
{
// if the spawned object is not a sim object itself, but if it's holding a ton of sim objects let's go
if (!resultObject.transform.GetComponent <SimObjPhysics>())
{
// each instantiated sliced version of the object is a bunch of sim objects held by a master parent transform, so go into each one and assign the id to each based on the parent's id so
// there is an association with the original source object
int count = 0;
foreach (Transform t in resultObject.transform)
{
SimObjPhysics tsop = t.GetComponent <SimObjPhysics>();
psm.Generate_InheritedObjectID(gameObject.GetComponent <SimObjPhysics>(), tsop, count);
count++;
// also turn on the kinematics of this object
Rigidbody trb = t.GetComponent <Rigidbody>();
trb.collisionDetectionMode = CollisionDetectionMode.ContinuousSpeculative;
trb.isKinematic = false;
// also add each child object's rb to the cache of all rigidbodies in scene
psm.AddToRBSInScene(trb);
}
}
// the spawned object is a sim object itself, so make an ID for it
else
{
// quick if the result object is an egg hard set it's rotation because EGGS ARE WEIRD and are not the same form as their shelled version
if (resultObject.GetComponent <SimObjPhysics>().Type == SimObjType.EggCracked)
{
resultObject.transform.rotation = Quaternion.Euler(Vector3.zero);
}
SimObjPhysics resultsop = resultObject.GetComponent <SimObjPhysics>();
psm.Generate_InheritedObjectID(gameObject.GetComponent <SimObjPhysics>(), resultsop, 0);
Rigidbody resultrb = resultsop.GetComponent <Rigidbody>();
resultrb.collisionDetectionMode = CollisionDetectionMode.ContinuousSpeculative;
resultrb.isKinematic = false;
// also add the spawned object's RB to the cache of all rigidbodies in scene
psm.AddToRBSInScene(resultrb);
}
}
else
{
Debug.LogError("Physics Scene Manager object is missing from scene!");
}
// if image synthesis is active, make sure to update the renderers for image synthesis since now there are new objects with renderes in the scene
BaseFPSAgentController primaryAgent = GameObject.Find("PhysicsSceneManager").GetComponent <AgentManager>().ReturnPrimaryAgent();
if (primaryAgent.imageSynthesis)
{
if (primaryAgent.imageSynthesis.enabled)
{
primaryAgent.imageSynthesis.OnSceneChange();
}
}
}
public IEnumerator EmitFrame()
{
frameCounter += 1;
bool shouldRender = this.renderImage && serverSideScreenshot;
if (shouldRender)
{
// we should only read the screen buffer after rendering is complete
yield return(new WaitForEndOfFrame());
}
WWWForm form = new WWWForm();
MultiAgentMetadata multiMeta = new MultiAgentMetadata();
multiMeta.agents = new MetadataWrapper[this.agents.Count];
multiMeta.activeAgentId = this.activeAgentId;
multiMeta.sequenceId = this.currentSequenceId;
ThirdPartyCameraMetadata[] cameraMetadata = new ThirdPartyCameraMetadata[this.thirdPartyCameras.Count];
RenderTexture currentTexture = null;
JavaScriptInterface jsInterface = null;
if (shouldRender)
{
currentTexture = RenderTexture.active;
for (int i = 0; i < this.thirdPartyCameras.Count; i++)
{
ThirdPartyCameraMetadata cMetadata = new ThirdPartyCameraMetadata();
Camera camera = thirdPartyCameras.ToArray()[i];
cMetadata.thirdPartyCameraId = i;
cMetadata.position = camera.gameObject.transform.position;
cMetadata.rotation = camera.gameObject.transform.eulerAngles;
cameraMetadata[i] = cMetadata;
ImageSynthesis imageSynthesis = camera.gameObject.GetComponentInChildren <ImageSynthesis> () as ImageSynthesis;
addThirdPartyCameraImageForm(form, camera);
addImageSynthesisImageForm(form, imageSynthesis, this.renderDepthImage, "_depth", "image_thirdParty_depth");
addImageSynthesisImageForm(form, imageSynthesis, this.renderNormalsImage, "_normals", "image_thirdParty_normals");
addImageSynthesisImageForm(form, imageSynthesis, this.renderObjectImage, "_id", "image_thirdParty_image_ids");
addImageSynthesisImageForm(form, imageSynthesis, this.renderClassImage, "_class", "image_thirdParty_classes");
addImageSynthesisImageForm(form, imageSynthesis, this.renderClassImage, "_flow", "image_thirdParty_flow");//XXX fix this in a bit
}
}
for (int i = 0; i < this.agents.Count; i++)
{
BaseFPSAgentController agent = this.agents.ToArray() [i];
jsInterface = agent.GetComponent <JavaScriptInterface>();
MetadataWrapper metadata = agent.generateMetadataWrapper();
metadata.agentId = i;
// we don't need to render the agent's camera for the first agent
if (shouldRender)
{
addImageForm(form, agent);
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderDepthImage, "_depth", "image_depth");
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderNormalsImage, "_normals", "image_normals");
addObjectImageForm(form, agent, ref metadata);
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderClassImage, "_class", "image_classes");
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderFlowImage, "_flow", "image_flow");
metadata.thirdPartyCameras = cameraMetadata;
}
multiMeta.agents [i] = metadata;
}
if (shouldRender)
{
RenderTexture.active = currentTexture;
}
var serializedMetadata = Newtonsoft.Json.JsonConvert.SerializeObject(multiMeta);
#if UNITY_WEBGL
// JavaScriptInterface jsI = FindObjectOfType<JavaScriptInterface>();
// jsInterface.SendAction(new ServerAction(){action = "Test"});
if (jsInterface != null)
{
jsInterface.SendActionMetadata(serializedMetadata);
}
#endif
//form.AddField("metadata", JsonUtility.ToJson(multiMeta));
form.AddField("metadata", serializedMetadata);
form.AddField("token", robosimsClientToken);
#if !UNITY_WEBGL
if (synchronousHttp)
{
if (this.sock == null)
{
// Debug.Log("connecting to host: " + robosimsHost);
IPAddress host = IPAddress.Parse(robosimsHost);
IPEndPoint hostep = new IPEndPoint(host, robosimsPort);
this.sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
try {
this.sock.Connect(hostep);
}
catch (SocketException e) {
Debug.Log("Socket exception: " + e.ToString());
}
}
if (this.sock != null && this.sock.Connected)
{
byte[] rawData = form.data;
string request = "POST /train HTTP/1.1\r\n" +
"Content-Length: " + rawData.Length.ToString() + "\r\n";
foreach (KeyValuePair <string, string> entry in form.headers)
{
request += entry.Key + ": " + entry.Value + "\r\n";
}
request += "\r\n";
int sent = this.sock.Send(Encoding.ASCII.GetBytes(request));
sent = this.sock.Send(rawData);
// waiting for a frame here keeps the Unity window in sync visually
// its not strictly necessary, but allows the interact() command to work properly
// and does not reduce the overall FPS
yield return(new WaitForEndOfFrame());
byte[] headerBuffer = new byte[1024];
int bytesReceived = 0;
byte[] bodyBuffer = null;
int bodyBytesReceived = 0;
int contentLength = 0;
// read header
while (true)
{
int received = this.sock.Receive(headerBuffer, bytesReceived, headerBuffer.Length - bytesReceived, SocketFlags.None);
if (received == 0)
{
Debug.LogError("0 bytes received attempting to read header - connection closed");
break;
}
bytesReceived += received;;
string headerMsg = Encoding.ASCII.GetString(headerBuffer, 0, bytesReceived);
int offset = headerMsg.IndexOf("\r\n\r\n");
if (offset > 0)
{
contentLength = parseContentLength(headerMsg.Substring(0, offset));
bodyBuffer = new byte[contentLength];
bodyBytesReceived = bytesReceived - (offset + 4);
Array.Copy(headerBuffer, offset + 4, bodyBuffer, 0, bodyBytesReceived);
break;
}
}
// read body
while (bodyBytesReceived < contentLength)
{
// check for 0 bytes received
int received = this.sock.Receive(bodyBuffer, bodyBytesReceived, bodyBuffer.Length - bodyBytesReceived, SocketFlags.None);
if (received == 0)
{
Debug.LogError("0 bytes received attempting to read body - connection closed");
break;
}
bodyBytesReceived += received;
//Debug.Log("total bytes received: " + bodyBytesReceived);
}
string msg = Encoding.ASCII.GetString(bodyBuffer, 0, bodyBytesReceived);
ProcessControlCommand(msg);
}
}
else
{
using (var www = UnityWebRequest.Post("http://" + robosimsHost + ":" + robosimsPort + "/train", form))
{
yield return(www.SendWebRequest());
if (www.isNetworkError || www.isHttpError)
{
Debug.Log("Error: " + www.error);
yield break;
}
ProcessControlCommand(www.downloadHandler.text);
}
}
#endif
}
private void addObjectImageForm(WWWForm form, BaseFPSAgentController agent, ref MetadataWrapper metadata)
{
if (this.renderObjectImage)
{
if (!agent.imageSynthesis.hasCapturePass("_id"))
{
Debug.LogError("Object Image not available in imagesynthesis - returning empty image");
}
byte[] bytes = agent.imageSynthesis.Encode("_id");
form.AddBinaryData("image_ids", bytes);
Color[] id_image = agent.imageSynthesis.tex.GetPixels();
Dictionary <Color, int[]> colorBounds = new Dictionary <Color, int[]> ();
for (int yy = 0; yy < tex.height; yy++)
{
for (int xx = 0; xx < tex.width; xx++)
{
Color colorOn = id_image [yy * tex.width + xx];
if (!colorBounds.ContainsKey(colorOn))
{
colorBounds [colorOn] = new int[] { xx, yy, xx, yy };
}
else
{
int[] oldPoint = colorBounds [colorOn];
if (xx < oldPoint [0])
{
oldPoint [0] = xx;
}
if (yy < oldPoint [1])
{
oldPoint [1] = yy;
}
if (xx > oldPoint [2])
{
oldPoint [2] = xx;
}
if (yy > oldPoint [3])
{
oldPoint [3] = yy;
}
}
}
}
List <ColorBounds> boundsList = new List <ColorBounds> ();
foreach (Color key in colorBounds.Keys)
{
ColorBounds bounds = new ColorBounds();
bounds.color = new ushort[] {
(ushort)Math.Round(key.r * 255),
(ushort)Math.Round(key.g * 255),
(ushort)Math.Round(key.b * 255)
};
bounds.bounds = colorBounds [key];
boundsList.Add(bounds);
}
metadata.colorBounds = boundsList.ToArray();
List <ColorId> colors = new List <ColorId> ();
foreach (Color key in agent.imageSynthesis.colorIds.Keys)
{
ColorId cid = new ColorId();
cid.color = new ushort[] {
(ushort)Math.Round(key.r * 255),
(ushort)Math.Round(key.g * 255),
(ushort)Math.Round(key.b * 255)
};
cid.name = agent.imageSynthesis.colorIds [key];
colors.Add(cid);
}
metadata.colors = colors.ToArray();
}
}
private IEnumerator EmitFrame()
{
frameCounter += 1;
// we should only read the screen buffer after rendering is complete
yield return(new WaitForEndOfFrame());
WWWForm form = new WWWForm();
MultiAgentMetadata multiMeta = new MultiAgentMetadata();
multiMeta.agents = new MetadataWrapper[this.agents.Count];
ThirdPartyCameraMetadata[] cameraMetadata = new ThirdPartyCameraMetadata[this.thirdPartyCameras.Count];
multiMeta.activeAgentId = this.activeAgentId;
multiMeta.sequenceId = this.currentSequenceId;
RenderTexture currentTexture = RenderTexture.active;
for (int i = 0; i < this.thirdPartyCameras.Count; i++)
{
ThirdPartyCameraMetadata cMetadata = new ThirdPartyCameraMetadata();
Camera camera = thirdPartyCameras.ToArray()[i];
cMetadata.thirdPartyCameraId = i;
cMetadata.position = camera.gameObject.transform.position;
cMetadata.rotation = camera.gameObject.transform.eulerAngles;
cameraMetadata[i] = cMetadata;
ImageSynthesis imageSynthesis = camera.gameObject.GetComponentInChildren <ImageSynthesis> () as ImageSynthesis;
addThirdPartyCameraImageForm(form, camera);
addImageSynthesisImageForm(form, imageSynthesis, this.renderDepthImage, "_depth", "image_thirdParty_depth");
addImageSynthesisImageForm(form, imageSynthesis, this.renderNormalsImage, "_normals", "image_thirdParty_normals");
addImageSynthesisImageForm(form, imageSynthesis, this.renderObjectImage, "_id", "image_thirdParty_image_ids");
addImageSynthesisImageForm(form, imageSynthesis, this.renderClassImage, "_class", "image_thirdParty_classes");
}
for (int i = 0; i < this.agents.Count; i++)
{
BaseFPSAgentController agent = this.agents.ToArray() [i];
if (i > 0)
{
this.agents.ToArray() [i - 1].m_Camera.enabled = false;
}
agent.m_Camera.enabled = true;
MetadataWrapper metadata = agent.generateMetadataWrapper();
metadata.agentId = i;
// we don't need to render the agent's camera for the first agent
addImageForm(form, agent);
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderDepthImage, "_depth", "image_depth");
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderNormalsImage, "_normals", "image_normals");
addObjectImageForm(form, agent, ref metadata);
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderClassImage, "_class", "image_classes");
metadata.thirdPartyCameras = cameraMetadata;
multiMeta.agents [i] = metadata;
}
if (this.agents.Count != 1)
{
this.agents.ToArray()[this.agents.Count - 1].m_Camera.enabled = false;
}
this.agents.ToArray()[0].m_Camera.enabled = true;
RenderTexture.active = currentTexture;
//form.AddField("metadata", JsonUtility.ToJson(multiMeta));
form.AddField("metadata", Newtonsoft.Json.JsonConvert.SerializeObject(multiMeta));
form.AddField("token", robosimsClientToken);
#if !UNITY_WEBGL
using (var www = UnityWebRequest.Post("http://" + robosimsHost + ":" + robosimsPort + "/train", form))
{
yield return(www.SendWebRequest());
if (www.isNetworkError || www.isHttpError)
{
Debug.Log("Error: " + www.error);
yield break;
}
ProcessControlCommand(www.downloadHandler.text);
}
#endif
}
public void BreakObject(Collision collision)
{
// prefab swap will switch the entire object out with a new prefab object entirely
if (breakType == BreakType.PrefabSwap)
{
// Disable this game object and spawn in the broken pieces
Rigidbody rb = gameObject.GetComponent <Rigidbody>();
// turn off everything except the top object
foreach (Transform t in gameObject.transform)
{
t.gameObject.SetActive(false);
}
// spawn in correct prefab to swap to at object's last location and rotation
// make sure to change to the correct variant of Prefab if the object isDirty
// if gameObject.GetComponent<Dirty>() - first check to make sure if this object can become dirty
// if object is dirty - probably get this from the "Dirty" component to keep everything nice and self contained
// PrefabToSwapTo = DirtyPrefabToSwapTo
if (gameObject.GetComponent <Dirty>())
{
// if the object is not clean, swap to the dirty prefab
if (gameObject.GetComponent <Dirty>().IsDirty())
{
PrefabToSwapTo = DirtyPrefabToSwapTo;
}
}
GameObject resultObject = Instantiate(PrefabToSwapTo, transform.position, transform.rotation);
broken = true;
// ContactPoint cp = collision.GetContact(0);
foreach (Rigidbody subRb in resultObject.GetComponentsInChildren <Rigidbody>())
{
subRb.velocity = rb.velocity * 0.4f;
subRb.angularVelocity = rb.angularVelocity * 0.4f;
}
rb.collisionDetectionMode = CollisionDetectionMode.Discrete;
rb.isKinematic = true;
// if this object breaking is an egg, set rotation for the EggCracked object
// quick if the result object is an egg hard set it's rotation because EGGS ARE WEIRD and are not the same form as their shelled version
if (resultObject.GetComponent <SimObjPhysics>())
{
if (resultObject.GetComponent <SimObjPhysics>().Type == SimObjType.EggCracked)
{
resultObject.transform.rotation = Quaternion.Euler(Vector3.zero);
PhysicsSceneManager psm = GameObject.Find("PhysicsSceneManager").GetComponent <PhysicsSceneManager>();
psm.Generate_InheritedObjectID(gameObject.GetComponent <SimObjPhysics>(), resultObject.GetComponent <SimObjPhysics>(), 0);
Rigidbody resultrb = resultObject.GetComponent <Rigidbody>();
resultrb.collisionDetectionMode = CollisionDetectionMode.ContinuousSpeculative;
resultrb.isKinematic = false;
}
}
// it's broken, make sure that it cant trigger this call again
readytobreak = false;
}
// if decal type, do not switch out the object but instead swap materials to show cracked/broken parts
if (breakType == BreakType.MaterialSwap)
{
// decal logic here
if (MaterialSwapObjects.Length > 0)
{
for (int i = 0; i < MaterialSwapObjects.Length; i++)
{
MaterialSwapObjects[i].MyObject.GetComponent <MeshRenderer>().materials = MaterialSwapObjects[i].OnMaterials;
}
}
// if the object can be toggled on/off, if it is on, turn it off since it is now broken
if (gameObject.GetComponent <CanToggleOnOff>())
{
gameObject.GetComponent <CanToggleOnOff>().isOn = false;
}
broken = true;
// it's broken, make sure that it cant trigger this call again
readytobreak = false;
}
if (breakType == BreakType.Decal)
{
// move shattered decal to location of the collision, or if there was no collision and this is being called
// directly from the Break() action, create a default decal i guess?
BreakForDecalType(collision);
}
BaseFPSAgentController primaryAgent = GameObject.Find("PhysicsSceneManager").GetComponent <AgentManager>().ReturnPrimaryAgent();
if (primaryAgent.imageSynthesis)
{
if (primaryAgent.imageSynthesis.enabled)
{
primaryAgent.imageSynthesis.OnSceneChange();
}
}
}
private IEnumerator EmitFrame()
{
frameCounter += 1;
bool shouldRender = this.renderImage && serverSideScreenshot;
if (shouldRender)
{
// we should only read the screen buffer after rendering is complete
yield return(new WaitForEndOfFrame());
if (synchronousHttp)
{
// must wait an additional frame when in synchronous mode otherwise the frame lags
yield return(new WaitForEndOfFrame());
}
}
WWWForm form = new WWWForm();
MultiAgentMetadata multiMeta = new MultiAgentMetadata();
multiMeta.agents = new MetadataWrapper[this.agents.Count];
multiMeta.activeAgentId = this.activeAgentId;
multiMeta.sequenceId = this.currentSequenceId;
ThirdPartyCameraMetadata[] cameraMetadata = new ThirdPartyCameraMetadata[this.thirdPartyCameras.Count];
RenderTexture currentTexture = null;
if (shouldRender)
{
currentTexture = RenderTexture.active;
for (int i = 0; i < this.thirdPartyCameras.Count; i++)
{
ThirdPartyCameraMetadata cMetadata = new ThirdPartyCameraMetadata();
Camera camera = thirdPartyCameras.ToArray()[i];
cMetadata.thirdPartyCameraId = i;
cMetadata.position = camera.gameObject.transform.position;
cMetadata.rotation = camera.gameObject.transform.eulerAngles;
cameraMetadata[i] = cMetadata;
ImageSynthesis imageSynthesis = camera.gameObject.GetComponentInChildren <ImageSynthesis> () as ImageSynthesis;
addThirdPartyCameraImageForm(form, camera);
addImageSynthesisImageForm(form, imageSynthesis, this.renderDepthImage, "_depth", "image_thirdParty_depth");
addImageSynthesisImageForm(form, imageSynthesis, this.renderNormalsImage, "_normals", "image_thirdParty_normals");
addImageSynthesisImageForm(form, imageSynthesis, this.renderObjectImage, "_id", "image_thirdParty_image_ids");
addImageSynthesisImageForm(form, imageSynthesis, this.renderClassImage, "_class", "image_thirdParty_classes");
}
}
for (int i = 0; i < this.agents.Count; i++)
{
BaseFPSAgentController agent = this.agents.ToArray() [i];
MetadataWrapper metadata = agent.generateMetadataWrapper();
metadata.agentId = i;
// we don't need to render the agent's camera for the first agent
if (shouldRender)
{
addImageForm(form, agent);
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderDepthImage, "_depth", "image_depth");
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderNormalsImage, "_normals", "image_normals");
addObjectImageForm(form, agent, ref metadata);
addImageSynthesisImageForm(form, agent.imageSynthesis, this.renderClassImage, "_class", "image_classes");
metadata.thirdPartyCameras = cameraMetadata;
}
multiMeta.agents [i] = metadata;
}
if (shouldRender)
{
RenderTexture.active = currentTexture;
}
//form.AddField("metadata", JsonUtility.ToJson(multiMeta));
form.AddField("metadata", Newtonsoft.Json.JsonConvert.SerializeObject(multiMeta));
form.AddField("token", robosimsClientToken);
#if !UNITY_WEBGL
if (synchronousHttp)
{
if (this.sock == null)
{
// Debug.Log("connecting to host: " + robosimsHost);
IPAddress host = IPAddress.Parse(robosimsHost);
IPEndPoint hostep = new IPEndPoint(host, robosimsPort);
try
{
Socket s = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
s.Connect(hostep);
this.sock = s;
}
catch (SocketException ex)
{
# if UNITY_EDITOR
// swallowing the error since its fine to run without the Python side with the editor
yield break;
# else
throw ex;
# endif
}
}
