/// <summary>
/// Runs a localization session on the specific <see cref="SpatialCoordinateSystemParticipant"/>'s connected peer, followed
/// by creating a persisted WorldAnchor-based <see cref="ISpatialCoordinate"/> based on the located coordinate.
/// </summary>
/// <param name="participant">The participant to use to initiate the remote localization sessions.</param>
/// <param name="spatialLocalizerId">The ID of the <see cref="ISpatialLocalizer"/> to use
/// for discovering a spatial coordinate.</param>
/// <param name="settings">The settings to pass to the remote localizer.</param>
public async void RunRemoteLocalizationWithWorldAnchorPersistence(SpatialCoordinateSystemParticipant participant, Guid spatialLocalizerId, ISpatialLocalizationSettings settings)
{
// If the initial request to restore a coordinate from a WorldAnchor hasn't completed, wait for that to complete first.
if (participantLocalizationTasks.TryGetValue(participant, out Task <bool> currentTask))
{
await currentTask;
}
// Request localization using the specific localizer and settings, and wait for that localization to complete.
participantLocalizationTasks[participant] = currentTask = SpatialCoordinateSystemManager.Instance.RunRemoteLocalizationAsync(participant.SocketEndpoint, spatialLocalizerId, settings);
bool localizationSucceeded = await currentTask;
if (localizationSucceeded)
{
// Once the specific localizer has found a shared coordinate, ask the WorldAnchorSpatialLocalizer
// to create a WorldAnchor-based coordinate at the same location, and persist that coordinate across sessions.
participantLocalizationTasks[participant] = currentTask = SpatialCoordinateSystemManager.Instance.RunRemoteLocalizationAsync(participant.SocketEndpoint, WorldAnchorSpatialLocalizer.Id, new WorldAnchorSpatialLocalizationSettings
{
Mode = WorldAnchorLocalizationMode.CreateAnchorAtWorldTransform,
AnchorId = CompositorWorldAnchorId,
AnchorPosition = participant.PeerSpatialCoordinateWorldPosition,
AnchorRotation = participant.PeerSpatialCoordinateWorldRotation
});
await currentTask;
}
else
{
Debug.LogError($"Remote localization failed on device {participant.SocketEndpoint.Address} for spatial localizer {spatialLocalizerId}");
}
}
private bool TryCleanupExistingLocalizationSession(SpatialCoordinateSystemParticipant participant)
{
bool succeeded = true;
LocalizationSessionDetails sessionToCancel = null;
lock (localizationLock)
{
if (currentLocalizationSession != null)
{
// When connecting to the StationaryCameraBroadcaster within the editor and performing spatial coordinate sharing through it,
// both sessions are running within the editor. Completing one part of the session will leave a completed task assigned to
// the remote participant within the same session. That session should be cleaned up here to allow the local session
// to start.
if (currentLocalizationSession.Participant == participant || currentLocalizationSession.CompletionSource.Task.IsCompleted)
{
sessionToCancel = currentLocalizationSession;
currentLocalizationSession = null;
}
else
{
succeeded = false;
}
}
}
if (sessionToCancel != null)
{
DebugLog($"Cancelling spatial localization session for participant: {participant?.NetworkConnection?.ToString() ?? "Unknown"}");
sessionToCancel.Session.Cancel();
sessionToCancel.Session.Dispose();
sessionToCancel.CompletionSource.TrySetResult(false);
}
return(succeeded);
}
private void OnConnected(INetworkConnection connection)
{
if (participants.TryGetValue(connection, out var existingParticipant))
{
Debug.LogWarning("SpatialCoordinateSystemParticipant connected that already existed.");
return;
}
DebugLog($"Creating new SpatialCoordinateSystemParticipant, NetworkConnection: {connection.ToString()}, DebugLogging: {debugLogging}");
SpatialCoordinateSystemParticipant participant = new SpatialCoordinateSystemParticipant(connection, debugVisual, debugVisualScale);
participants[connection] = participant;
participant.ShowDebugVisuals = showDebugVisuals;
participant.SendSupportedLocalizersMessage(connection, localizers.Keys);
if (ParticipantConnected == null)
{
DebugLog($"No ParticipantConnected event handlers exist");
}
else
{
DebugLog($"Invoking ParticipantConnected event");
ParticipantConnected.Invoke(participant);
}
}
private void OnConnected(SocketEndpoint endpoint)
{
if (participants.TryGetValue(endpoint, out var existingParticipant))
{
Debug.LogWarning("SpatialCoordinateSystemParticipant connected that already existed.");
return;
}
DebugLog($"Creating new SpatialCoordinateSystemParticipant, IPAddress: {endpoint.Address}, DebugLogging: {debugLogging}");
SpatialCoordinateSystemParticipant participant = new SpatialCoordinateSystemParticipant(endpoint, debugVisual, debugVisualScale);
participants[endpoint] = participant;
participant.ShowDebugVisuals = showDebugVisuals;
participant.SendSupportedLocalizersMessage(endpoint, localizers.Keys);
if (ParticipantConnected == null)
{
DebugLog($"No ParticipantConnected event handlers exist");
}
else
{
DebugLog($"Invoking ParticipantConnected event");
ParticipantConnected.Invoke(participant);
}
}
private void NetworkManagerConnected(INetworkConnection connection)
{
// Restart the timeline at 0 each time we reconnect to the HoloLens
timestampStopwatch = Stopwatch.StartNew();
sharedCoordinateParticipant = null;
currentConnection = connection;
}
private void OnConnected(SocketEndpoint endpoint)
{
if (participants.ContainsKey(endpoint))
{
Debug.LogWarning("SpatialCoordinateSystemParticipant connected that already existed");
return;
}
DebugLog($"Creating new SpatialCoordinateSystemParticipant, IPAddress: {endpoint.Address}, DebugLogging: {debugLogging}");
SpatialCoordinateSystemParticipant participant = new SpatialCoordinateSystemParticipant(endpoint, debugVisual, debugVisualScale);
participants[endpoint] = participant;
participant.ShowDebugVisuals = showDebugVisuals;
if (ParticipantConnected == null)
{
Debug.LogWarning("Participant created, but no connection listeners were found");
return;
}
participant.SendSupportedLocalizersMessage(endpoint, localizers.Keys);
DebugLog($"Invoking ParticipantConnected event");
ParticipantConnected.Invoke(participant);
}
private async void OnParticipantConnected(SpatialCoordinateSystemParticipant participant)
{
DebugLog($"Waiting for the set of supported localizers from connected participant {participant.SocketEndpoint.Address}");
// When a remote participant connects, get the set of ISpatialLocalizers that peer
// supports. This is asynchronous, as it comes across the network.
ISet <Guid> peerSupportedLocalizers = await participant.GetPeerSupportedLocalizersAsync();
// If there are any supported localizers, find the first configured localizer in the
// list that supports that type. If and when one is found, use it to perform localization.
if (peerSupportedLocalizers != null)
{
DebugLog($"Received a set of {peerSupportedLocalizers.Count} supported localizers");
if (SpatialLocalizationInitializationSettings.IsInitialized &&
TryRunLocalization(SpatialLocalizationInitializationSettings.Instance.PrioritizedInitializers, peerSupportedLocalizers, participant))
{
// Succeeded at using a custom localizer specified by the app.
return;
}
if (TryRunLocalization(defaultSpatialLocalizationInitializers, peerSupportedLocalizers, participant))
{
// Succeeded at using one of the default localizers from the prefab.
return;
}
Debug.LogWarning($"None of the configured LocalizationInitializers were supported by the connected participant, localization will not be started");
}
else
{
Debug.LogWarning($"No supported localizers were received from the participant, localization will not be started");
}
}
private void TcpConnectionManager_OnConnected(SocketEndpoint endpoint)
{
// Restart the timeline at 0 each time we reconnect to the HoloLens
timestampStopwatch = Stopwatch.StartNew();
sharedCoordinateParticipant = null;
currentConnection = endpoint;
}
private bool TryCleanupExistingLocalizationSession(SpatialCoordinateSystemParticipant participant)
{
bool succeeded = true;
LocalizationSessionDetails sessionToCancel = null;
lock (localizationLock)
{
if (currentLocalizationSession != null)
{
if (currentLocalizationSession.Participant == participant)
{
sessionToCancel = currentLocalizationSession;
currentLocalizationSession = null;
}
else
{
succeeded = false;
}
}
}
if (sessionToCancel != null)
{
DebugLog($"Cancelling spatial localization session for participant: {participant?.SocketEndpoint?.Address ?? "Unknown"}");
sessionToCancel.Session.Cancel();
sessionToCancel.Session.Dispose();
sessionToCancel.CompletionSource.TrySetResult(false);
}
return(succeeded);
}
public override void RunLocalization(SpatialCoordinateSystemParticipant participant)
{
SpatialCoordinateSystemManager.Instance.LocalizeAsync(participant.SocketEndpoint, SpatialAnchorsLocalizer.Id, configuration);
configuration.IsCoordinateCreator = true;
SpatialCoordinateSystemManager.Instance.RunRemoteLocalizationAsync(participant.SocketEndpoint, SpatialAnchorsLocalizer.Id, configuration);
}
public override void RunLocalization(SpatialCoordinateSystemParticipant participant)
{
DebugLog($"Marker-based localization started for: {participant?.SocketEndpoint?.Address ?? "IPAddress unknown"} with marker type {markerType}");
// Note: We need to send the remote localization message prior to starting marker visual localization. The MarkerVisualSpatialLocalizer won't return until localization has completed.
SpatialCoordinateSystemManager.Instance.RunRemoteLocalizationAsync(participant.SocketEndpoint, PeerSpatialLocalizerId, new MarkerVisualDetectorLocalizationSettings());
SpatialCoordinateSystemManager.Instance.LocalizeAsync(participant.SocketEndpoint, LocalSpatialLocalizerId, new MarkerVisualLocalizationSettings());
}
private void OnParticipantConnected(SpatialCoordinateSystemParticipant participant)
{
DebugLog($"Participant connected: {participant?.NetworkConnection?.ToString() ?? "Unknown NetworkConnection"}");
if (currentParticipant != null)
{
DebugLog("Participant was already registered when new participant connected");
}
currentParticipant = participant;
}
private void OnParticipantDisconnected(SpatialCoordinateSystemParticipant participant)
{
DebugLog($"Participant disconnected: {participant?.NetworkConnection?.ToString() ?? "Unknown NetworkConnection"}");
if (currentParticipant == null)
{
DebugLog("No participant was registered when a participant disconnected");
}
currentParticipant = null;
}
private void OnParticipantConnected(SpatialCoordinateSystemParticipant participant)
{
// When a new participant connects, send a request to re-load the shared spatial coordinate world anchor.
participantLocalizationTasks.Add(participant, SpatialCoordinateSystemManager.Instance.RunRemoteLocalizationAsync(participant.SocketEndpoint, WorldAnchorSpatialLocalizer.Id, new WorldAnchorSpatialLocalizationSettings
{
Mode = WorldAnchorLocalizationMode.LocateExistingAnchor,
AnchorId = CompositorWorldAnchorId
}));
}
private void OnParticipantConnected(SpatialCoordinateSystemParticipant participant)
{
DebugLog($"Participant connected: {participant?.SocketEndpoint?.Address ?? "IPAddress unknown"}");
if (currentParticipant != null)
{
DebugLog("Participant was already registered when new participant connected");
}
currentParticipant = participant;
}
private void OnParticipantConnected(SpatialCoordinateSystemParticipant participant)
{
DebugLog($"Participant connected: {participant?.SocketEndpoint?.Address ?? "IPAddress unknown"}");
if (currentParticipant != null)
{
Debug.LogError("Unexpected existing participant when another participant connected");
}
currentParticipant = participant;
}
private void OnParticipantDisconnected(SpatialCoordinateSystemParticipant participant)
{
DebugLog($"Participant disconnected: {participant?.SocketEndpoint?.Address ?? "IPAddress unknown"}");
if (currentParticipant == null)
{
DebugLog("No participant was registered when a participant disconnected");
}
currentParticipant = null;
}
private async Task <bool> TryRunLocalizationImplAsync(SpatialCoordinateSystemParticipant participant)
{
DebugLog($"Marker-based localization started for: {participant?.SocketEndpoint?.Address ?? "IPAddress unknown"} with marker type {markerType}");
// Note: We need to send the remote localization message prior to starting marker visual localization. The MarkerVisualSpatialLocalizer won't return until localization has completed.
Task <bool> remoteTask = SpatialCoordinateSystemManager.Instance.RunRemoteLocalizationAsync(participant.SocketEndpoint, PeerSpatialLocalizerId, new MarkerVisualDetectorLocalizationSettings());
Task <bool> localTask = SpatialCoordinateSystemManager.Instance.LocalizeAsync(participant.SocketEndpoint, LocalSpatialLocalizerId, new MarkerVisualLocalizationSettings());
await Task.WhenAll(remoteTask, localTask);
bool localSuccess = await localTask;
bool remoteSuccess = await remoteTask;
return(localSuccess && remoteSuccess);
}
private async Task <bool> TryRunLocalizationImplAsync(SpatialCoordinateSystemParticipant participant)
{
configuration.IsCoordinateCreator = false;
Task <bool> localTask = SpatialCoordinateSystemManager.Instance.LocalizeAsync(participant.NetworkConnection, SpatialAnchorsLocalizer.Id, configuration);
configuration.IsCoordinateCreator = true;
Task <bool> remoteTask = SpatialCoordinateSystemManager.Instance.RunRemoteLocalizationAsync(participant.NetworkConnection, SpatialAnchorsLocalizer.Id, configuration);
await Task.WhenAll(localTask, remoteTask);
bool localSuccess = await localTask;
bool remoteSuccess = await remoteTask;
return(localSuccess && remoteSuccess);
}
private async Task <bool> ConditionallyRunWorldAnchorLocalizationAsync(SpatialCoordinateSystemParticipant participant)
{
var supportedLocalizers = await participant.GetPeerSupportedLocalizersAsync();
if (!supportedLocalizers.Contains(WorldAnchorSpatialLocalizer.Id))
{
return(false);
}
return(await SpatialCoordinateSystemManager.Instance.RunRemoteLocalizationAsync(participant.NetworkConnection, WorldAnchorSpatialLocalizer.Id, new WorldAnchorSpatialLocalizationSettings
{
Mode = WorldAnchorLocalizationMode.LocateExistingAnchor,
AnchorId = CompositorWorldAnchorId
}));
}
private async Task <bool> TryRunLocalizationForParticipantAsync(SpatialCoordinateSystemParticipant participant)
{
DebugLog($"Waiting for the set of supported localizers from connected participant {participant.SocketEndpoint.Address}");
if (!peerSupportedLocalizers.ContainsKey(participant))
{
// When a remote participant connects, get the set of ISpatialLocalizers that peer
// supports. This is asynchronous, as it comes across the network.
peerSupportedLocalizers[participant] = await participant.GetPeerSupportedLocalizersAsync();
}
// If there are any supported localizers, find the first configured localizer in the
// list that supports that type. If and when one is found, use it to perform localization.
if (peerSupportedLocalizers.TryGetValue(participant, out var supportedLocalizers) &&
supportedLocalizers != null)
{
DebugLog($"Received a set of {peerSupportedLocalizers.Count} supported localizers");
var initializers = new List <SpatialLocalizationInitializer>();
if (SpatialLocalizationInitializationSettings.IsInitialized &&
SpatialLocalizationInitializationSettings.Instance.PrioritizedInitializers != null)
{
DebugLog($"Found prioritized spatial localization initializers list in spectator view settings");
foreach (var initializer in SpatialLocalizationInitializationSettings.Instance.PrioritizedInitializers)
{
initializers.Add(initializer);
}
}
if (defaultSpatialLocalizationInitializers != null)
{
DebugLog($"Found default spatial localization initializers list for spectator view settings");
foreach (var initializer in defaultSpatialLocalizationInitializers)
{
initializers.Add(initializer);
}
}
return(await TryRunLocalizationWithInitializerAsync(initializers, supportedLocalizers, participant));
}
else
{
Debug.LogWarning($"No supported localizers were received from the participant, localization will not be started");
}
return(false);
}
private async Task <bool> TryRunLocalizationForParticipantAsync(SpatialCoordinateSystemParticipant participant)
{
DebugLog($"Waiting for the set of supported localizers from connected participant {participant.SocketEndpoint.Address}");
if (!peerSupportedLocalizers.ContainsKey(participant))
{
// When a remote participant connects, get the set of ISpatialLocalizers that peer
// supports. This is asynchronous, as it comes across the network.
peerSupportedLocalizers[participant] = await participant.GetPeerSupportedLocalizersAsync();
}
// If there are any supported localizers, find the first configured localizer in the
// list that supports that type. If and when one is found, use it to perform localization.
if (peerSupportedLocalizers.TryGetValue(participant, out var supportedLocalizers) &&
supportedLocalizers != null)
{
DebugLog($"Received a set of {peerSupportedLocalizers.Count} supported localizers");
DebugLog($"Using prioritized initializers list from spectator view settings: {SpatialLocalizationInitializationSettings.IsInitialized && SpatialLocalizationInitializationSettings.Instance.PrioritizedInitializers != null}");
if (SpatialLocalizationInitializationSettings.IsInitialized &&
await TryRunLocalizationWithInitializerAsync(SpatialLocalizationInitializationSettings.Instance.PrioritizedInitializers, supportedLocalizers, participant))
{
// Succeeded at using a custom localizer specified by the app.
return(true);
}
if (await TryRunLocalizationWithInitializerAsync(defaultSpatialLocalizationInitializers, supportedLocalizers, participant))
{
// Succeeded at using one of the default localizers from the prefab.
return(true);
}
Debug.LogWarning($"None of the configured LocalizationInitializers were supported by the connected participant, localization will not be started");
}
else
{
Debug.LogWarning($"No supported localizers were received from the participant, localization will not be started");
}
return(false);
}
public LocalizationSessionDetails(ISpatialLocalizationSession session, SpatialCoordinateSystemParticipant participant)
{
this.Session = session;
this.Participant = participant;
this.CompletionSource = new TaskCompletionSource <bool>();
}
/// <inheritdoc />
public override async Task <bool> TryResetLocalizationAsync(SpatialCoordinateSystemParticipant participant)
{
return(await TryRunLocalizationImplAsync(participant));
}
private void OnParticipantDisconnected(SpatialCoordinateSystemParticipant participant)
{
participantLocalizationTasks.Remove(participant);
}
public bool TryGetSpatialCoordinateSystemParticipant(SocketEndpoint connectedEndpoint, out SpatialCoordinateSystemParticipant participant)
{
return(participants.TryGetValue(connectedEndpoint, out participant));
}
private async Task <bool> RunLocalizationSessionAsync(ISpatialLocalizer localizer, ISpatialLocalizationSettings settings, SpatialCoordinateSystemParticipant participant)
{
DebugLog($"Creating localization session: {participant.SocketEndpoint.Address}, {settings.ToString()}, {localizer.ToString()}");
if (!localizer.TryCreateLocalizationSession(participant, settings, out ISpatialLocalizationSession currentLocalizationSession))
{
Debug.LogError($"Failed to create an ISpatialLocalizationSession from localizer {localizer.SpatialLocalizerId}");
return(false);
}
using (currentLocalizationSession)
{
DebugLog($"Setting localization session for participant: {participant.SocketEndpoint.Address}, {currentLocalizationSession.ToString()}");
participant.CurrentLocalizationSession = currentLocalizationSession;
try
{
DebugLog($"Starting localization: {participant.SocketEndpoint.Address}, {currentLocalizationSession.ToString()}");
// Some SpatialLocalizers/SpatialCoordinateServices key off of token cancellation for their logic flow.
// Therefore, we need to create a cancellation token even it is never actually cancelled by the SpatialCoordinateSystemManager.
using (var localizeCTS = new CancellationTokenSource())
{
var coordinate = await currentLocalizationSession.LocalizeAsync(localizeCTS.Token);
participant.Coordinate = coordinate;
}
}
finally
{
participant.CurrentLocalizationSession = null;
}
}
currentLocalizationSession = null;
return(participant.Coordinate != null);
}
private void OnParticipantConnected(SpatialCoordinateSystemParticipant participant)
{
currentParticipant = participant;
TryRunLocalizationForParticipantAsync(currentParticipant).FireAndForget();
}
private async Task <bool> TryRunLocalizationWithInitializerAsync(IList <SpatialLocalizationInitializer> initializers, ISet <Guid> supportedLocalizers, SpatialCoordinateSystemParticipant participant)
{
if (initializers == null || supportedLocalizers == null)
{
Debug.LogWarning("Did not find a supported localizer/initializer combination.");
return(false);
}
DebugLog($"TryRunLocalizationWithInitializerAsync, initializers:{initializers.Count}, supportedLocalizers:{supportedLocalizers.Count}");
for (int i = 0; i < initializers.Count; i++)
{
if (supportedLocalizers.Contains(initializers[i].PeerSpatialLocalizerId))
{
DebugLog($"Localization initializer {initializers[i].GetType().Name} supported localization with ID {initializers[i].PeerSpatialLocalizerId}, starting localization");
bool result = await initializers[i].TryRunLocalizationAsync(participant);
if (!result)
{
Debug.LogError($"Failed to localize experience with participant: {participant.SocketEndpoint.Address}");
}
else
{
DebugLog($"Succeeded in localizing experience with participant: {participant.SocketEndpoint.Address}");
}
return(result);
}
else
{
DebugLog($"Localization initializer {initializers[i].GetType().Name} not supported by peer.");
}
}
return(await Task.FromResult(false));
}
private async Task <bool> RunLocalizationSessionAsync(ISpatialLocalizer localizer, ISpatialLocalizationSettings settings, SpatialCoordinateSystemParticipant participant)
{
if (!TryCleanupExistingLocalizationSession(participant))
{
DebugLog("Existing localization session with different participant prevented creating new localization session");
return(false);
}
if (!localizer.TryCreateLocalizationSession(participant, settings, out var localizationSession))
{
Debug.LogError($"Failed to create an ISpatialLocalizationSession from localizer {localizer.SpatialLocalizerId}");
return(false);
}
Task <bool> resultTask;
bool startSession = false;
var localizationSessionDetails = new LocalizationSessionDetails(localizationSession, participant);
lock (localizationLock)
{
if (currentLocalizationSession != null)
{
DebugLog($"Current localization session repopulated after cleanup, localization not performed.");
localizationSessionDetails.Session.Dispose();
resultTask = Task.FromResult(false);
}
else
{
currentLocalizationSession = localizationSessionDetails;
localizationSessionDetails.Participant.CurrentLocalizationSession = localizationSessionDetails.Session;
resultTask = localizationSessionDetails.CompletionSource.Task;
startSession = true;
}
}
if (startSession)
{
await Dispatcher.ScheduleAsync(async() =>
{
try
{
// Some SpatialLocalizers/SpatialCoordinateServices key off of token cancellation for their logic flow.
// Therefore, we need to create a cancellation token even it is never actually cancelled by the SpatialCoordinateSystemManager.
using (var localizeCTS = new CancellationTokenSource())
{
var coordinate = await localizationSessionDetails.Session.LocalizeAsync(localizeCTS.Token);
bool succeeded = (coordinate != null);
localizationSessionDetails.Session.Dispose();
localizationSessionDetails.CompletionSource.TrySetResult(succeeded);
if (localizationSessionDetails.Participant.CurrentLocalizationSession == localizationSessionDetails.Session)
{
localizationSessionDetails.Participant.Coordinate = coordinate;
localizationSessionDetails.Participant.CurrentLocalizationSession = null;
}
else
{
Debug.LogWarning("Localization session completed but was no longer assigned to the associated participant");
}
}
}
catch (OperationCanceledException)
{
DebugLog("Localization operation cancelled.");
}
catch (Exception e)
{
Debug.LogError($"Exception thrown localizing experience: {e.ToString()}");
}
}, CancellationToken.None, true);
}
return(await resultTask);
}
