/// <summary>
/// Get the nearest <see cref="Edge"/> to a screen position.
/// </summary>
/// <returns>
/// Distance is returned as the screen distance to mesh, not edge.
/// </returns>
static float EdgeRaycast(Vector3 mousePosition, ScenePickerPreferences pickerPrefs, bool allowUnselected, SceneSelection selection)
{
selection.Clear();
selection.gameObject = UHandleUtility.PickGameObject(mousePosition, false);
var hoveredMesh = selection.gameObject != null?selection.gameObject.GetComponent <ProBuilderMesh>() : null;
float bestDistance = Mathf.Infinity;
bool hoveredIsInSelection = MeshSelection.topInternal.Contains(hoveredMesh);
if (hoveredMesh != null && (allowUnselected || hoveredIsInSelection))
{
var tup = GetNearestEdgeOnMesh(hoveredMesh, mousePosition);
if (tup.edge.IsValid())
{
selection.gameObject = hoveredMesh.gameObject;
selection.mesh = hoveredMesh;
selection.SetSingleEdge(tup.edge);
bestDistance = tup.distance;
// If the nearest edge was acquired by a raycast, then the distance to mesh is 0f.
if (hoveredIsInSelection)
{
return(tup.distance);
}
}
}
foreach (var mesh in MeshSelection.topInternal)
{
var trs = mesh.transform;
var positions = mesh.positionsInternal;
s_EdgeBuffer.Clear();
// When the pointer is over another object, apply a modifier to the distance to prefer picking the
// object hovered over the currently selected
var distMultiplier = (hoveredMesh == mesh || hoveredMesh == null)
? 1.0f
: ScenePickerPreferences.offPointerMultiplier;
foreach (var face in mesh.facesInternal)
{
foreach (var edge in face.edges)
{
int x = edge.a;
int y = edge.b;
float d = UHandleUtility.DistanceToLine(
trs.TransformPoint(positions[x]),
trs.TransformPoint(positions[y]));
d *= distMultiplier;
// best distance isn't set to maxPointerDistance because we want to preserve an unselected
// gameobject over a selected gameobject with an out of bounds edge.
if (d > ScenePickerPreferences.maxPointerDistance)
{
continue;
}
// account for stacked edges
if (Mathf.Approximately(d, bestDistance))
{
s_EdgeBuffer.Add(new Edge(x, y));
}
else if (d < bestDistance)
{
s_EdgeBuffer.Clear();
s_EdgeBuffer.Add(new Edge(x, y));
selection.gameObject = mesh.gameObject;
selection.mesh = mesh;
selection.SetSingleEdge(new Edge(x, y));
bestDistance = d;
}
}
}
// If more than 1 edge is closest, the closest is one of the vertex.
// Get closest edge to the camera.
if (s_EdgeBuffer.Count > 1)
{
selection.SetSingleEdge(GetClosestEdgeToCamera(positions, s_EdgeBuffer));
}
}
return(selection.gameObject != null ? bestDistance : Mathf.Infinity);
}
