public static bool VertexRaycast(Vector2 mousePosition, int rectSize, ElementCache selection, out int index)
{
qe_Mesh mesh = selection.mesh;
float bestDistance = Mathf.Infinity;
float distance = 0f;
index = -1;
GameObject go = HandleUtility.PickGameObject(mousePosition, false);
if (go == null || go != selection.transform.gameObject)
{
Camera cam = SceneView.lastActiveSceneView.camera;
int width = Screen.width;
int height = Screen.height;
Rect mouseRect = new Rect(mousePosition.x - (rectSize / 2f), mousePosition.y - (rectSize / 2f), rectSize, rectSize);
List <int> user = (List <int>)selection.mesh.GetUserIndices();
for (int i = 0; i < user.Count; i++)
{
if (mouseRect.Contains(HandleUtility.WorldToGUIPoint(selection.verticesInWorldSpace[user[i]])))
{
Vector3 v = cam.WorldToScreenPoint(selection.verticesInWorldSpace[user[i]]);
distance = Vector2.Distance(mousePosition, v);
if (distance < bestDistance)
{
if (v.z <= 0 || v.x < 0 || v.y < 0 || v.x > width || v.y > height)
{
continue;
}
if (PointIsOccluded(mesh, selection.verticesInWorldSpace[user[i]]))
{
continue;
}
index = user[i];
bestDistance = Vector2.Distance(v, mousePosition);
}
}
}
}
else
{
// Test culling
List <qe_RaycastHit> hits;
Ray ray = HandleUtility.GUIPointToWorldRay(mousePosition);
if (MeshRaycast(ray, mesh, out hits, Mathf.Infinity, Culling.FrontBack))
{
// Sort from nearest hit to farthest
hits.Sort((x, y) => x.Distance.CompareTo(y.Distance));
// Find the nearest edge in the hit faces
Vector3[] v = mesh.vertices;
for (int i = 0; i < hits.Count; i++)
{
if (PointIsOccluded(mesh, mesh.transform.TransformPoint(hits[i].Point)))
{
continue;
}
foreach (int tri in mesh.faces[hits[i].FaceIndex].indices)
{
float d = Vector3.Distance(hits[i].Point, v[tri]);
if (d < bestDistance)
{
bestDistance = d;
index = tri;
}
}
if (Vector3.Dot(ray.direction, mesh.transform.TransformDirection(hits[i].Normal)) < 0f)
{
break;
}
}
if (index > -1 && Vector2.Distance(mousePosition, HandleUtility.WorldToGUIPoint(selection.verticesInWorldSpace[index])) > rectSize * 1.3f)
{
index = -1;
}
}
}
if (index > -1)
{
index = mesh.ToUserIndex(index);
}
return(index > -1);
}
/**
* Checks if mouse is over an edge, and if so, returns true setting @edge.
*/
public static bool EdgeRaycast(Vector2 mousePosition, ElementCache selection, out qe_Edge edge)
{
qe_Mesh mesh = selection.mesh;
Vector3 v0, v1;
float bestDistance = Mathf.Infinity;
float distance = 0f;
edge = null;
GameObject go = HandleUtility.PickGameObject(mousePosition, false);
if (go == null || go != selection.transform.gameObject)
{
qe_Edge[] edges = mesh.userEdges;
int width = Screen.width;
int height = Screen.height;
for (int i = 0; i < edges.Length; i++)
{
v0 = selection.verticesInWorldSpace[edges[i].x];
v1 = selection.verticesInWorldSpace[edges[i].y];
distance = HandleUtility.DistanceToLine(v0, v1);
if (distance < bestDistance && distance < MAX_EDGE_SELECT_DISTANCE) // && !PointIsOccluded(mesh, (v0+v1)*.5f) )
{
Vector3 vs0 = SceneView.lastActiveSceneView.camera.WorldToScreenPoint(v0);
// really simple frustum check (will fail on edges that have vertices outside the frustum but is visible)
if (vs0.z <= 0 || vs0.x < 0 || vs0.y < 0 || vs0.x > width || vs0.y > height)
{
continue;
}
Vector3 vs1 = SceneView.lastActiveSceneView.camera.WorldToScreenPoint(v1);
if (vs1.z <= 0 || vs1.x < 0 || vs1.y < 0 || vs1.x > width || vs1.y > height)
{
continue;
}
bestDistance = distance;
edge = edges[i];
}
}
}
else
{
// Test culling
List <qe_RaycastHit> hits;
Ray ray = HandleUtility.GUIPointToWorldRay(mousePosition);
if (MeshRaycast(ray, mesh, out hits, Mathf.Infinity, Culling.FrontBack))
{
// Sort from nearest hit to farthest
hits.Sort((x, y) => x.Distance.CompareTo(y.Distance));
// Find the nearest edge in the hit faces
Vector3[] v = mesh.vertices;
for (int i = 0; i < hits.Count; i++)
{
if (PointIsOccluded(mesh, mesh.transform.TransformPoint(hits[i].Point)))
{
continue;
}
foreach (qe_Edge e in mesh.faces[hits[i].FaceIndex].GetEdges())
{
float d = HandleUtility.DistancePointLine(hits[i].Point, v[e.x], v[e.y]);
if (d < bestDistance)
{
bestDistance = d;
edge = e;
}
}
if (Vector3.Dot(ray.direction, mesh.transform.TransformDirection(hits[i].Normal)) < 0f)
{
break;
}
}
if (edge != null && HandleUtility.DistanceToLine(mesh.transform.TransformPoint(v[edge.x]), mesh.transform.TransformPoint(v[edge.y])) > MAX_EDGE_SELECT_DISTANCE)
{
edge = null;
}
else
{
edge.x = mesh.ToUserIndex(edge.x);
edge.y = mesh.ToUserIndex(edge.y);
}
}
}
return(edge != null);
}
