public void ToggleWater()
{
if (textToSpeech.IsSpeaking())
{
textToSpeech.StopSpeaking();
}
else
{
TellMeMore();
}
ToggleVisibility();
}
public void ResetPositions()
{
//Adjusts obstacles back to last saved position.
Debug.Log("Resetting positions.");
if (!textToSpeech.IsSpeaking())
{
textToSpeech.StartSpeaking("Positions reset.");
}
//int obstacleCount = 0;
int cueCount = 0;
/*
* foreach (GameObject Obstacle in digitalObstacles)
* {
* Obstacle.transform.position = obstacleStartingLocations[obstacleCount];
* Obstacle.transform.rotation = obstacleStartingRotations[obstacleCount];
* Obstacle.transform.localScale = obstacleStartingScales[obstacleCount];
* obstacleCount++;
* }
*/
foreach (GameObject Cue in visualCues)
{
Cue.transform.localPosition = cueStartingLocations[cueCount];
Cue.transform.localRotation = cueStartingRotations[cueCount];
Cue.transform.localScale = cueStartingScales[cueCount];
cueCount++;
}
}
// Update is called once per frame
void Update()
{
targetTime -= Time.deltaTime;
if (targetTime <= 0.0f && !textToSpeech.IsSpeaking())
{
if (state == 0)
{
++state;
string s = "Please follow the red target.";
Debug.Log(s);
textToSpeech.StartSpeaking(s);
targetTime = 1f;
}
}
if (targetTime <= 0.0f && state == 1)
{
//stimulusManager.GazeShift(trainingTarget);
stimulusManager.GazeShift();
if (EVENT_MARKER_TYPE == TCP_EVENT_MARKER)
{
#if !UNITY_EDITOR
Connect();
#endif
}
else if (EVENT_MARKER_TYPE == LPT_EVENT_MARKER)
{
ParallelPortManager.Out32_x64(ParallelPortManager.address, 0);
}
if (++trainingTarget == numPerColumn * numPerRow)
{
trainingTarget = 0;
}
targetTime = 1f;
state = 2;
}
if (targetTime <= 0.0f && state == 2)
{
byte[] msg = new byte[1];
msg[0] = 1;
stimulusManager.StartStimulating();
targetTime = 2f;
state = 1;
if (EVENT_MARKER_TYPE == TCP_EVENT_MARKER)
{
#if !UNITY_EDITOR
SendMessage(msg);
#endif
}
else if (EVENT_MARKER_TYPE == LPT_EVENT_MARKER)
{
ParallelPortManager.Out32_x64(ParallelPortManager.address, 1);
}
}
}
private void SayLastSeenObject()
{
if (_tts != null && !_tts.IsSpeaking())
{
_tts.StartSpeaking(_speechText);
}
}
public void StopVoice()
{
if (textToSpeech.IsSpeaking())
{
textToSpeech.StopSpeaking();
}
}
// <summary>
/// Read the text
/// </summary>
/// <param name="text">Text to be spoken.</param>
public static void Read(string text)
{
// Don't talk empty and don't talk over.
if (!string.IsNullOrEmpty(text) && !textToSpeech.IsSpeaking())
{
textToSpeech.StartSpeaking(text);
}
}
public override bool IsCompleted()
{
#if UNITY_EDITOR
return(true);
#else
return(_tts.IsSpeaking());
#endif
}
private async Task EvaluateFrame(Windows.Media.VideoFrame videoFrame)
{
try
{
var result = await _dnnModel.EvaluateVideoFrameAsync(videoFrame);
if (result.DominantResultProbability > 0)
{
// Further process and surface results to UI on the UI thread
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
// Measure distance between user's head and gaze ray hit point => distance to object
var distMessage = string.Empty;
if (_user.GazeHitDistance < 1)
{
distMessage = string.Format("{0:f0} {1}", _user.GazeHitDistance * 100, "centimeter");
}
else
{
distMessage = string.Format("{0:f1} {1}", _user.GazeHitDistance, "meter");
}
// Prepare strings for text and update labels
var deviceKind = ShouldUseGpu ? "GPU" : "CPU";
var labelText = $"Predominant objects detected in {result.ElapsedMilliseconds,3:f0}ms on {deviceKind}\n {result.TopResultsFormatted}";
var speechText = string.Format("This {0} a {1} {2} in front of you",
result.DominantResultProbability > ProbabilityThreshold ? "is likely" : "might be",
result.DominantResultLabel,
distMessage);
StatusBlock.text = labelText;
// Check if the previous result was the same and only progress further if not to avoid a loop of same audio
if (!_tts.IsSpeaking() && result.DominantResultLabel != _previousDominantResult)
{
_tts.StartSpeaking(speechText);
_previousDominantResult = result.DominantResultLabel;
}
}, false);
}
}
catch (Exception ex)
{
//IsRunning = false;
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
//StatusBlock.text = $"Error loop: {ex.Message}";
//Debug.LogError(ex);
//Debug.LogError(videoFrame.Direct3DSurface == null ? "D3D null" : "D3D set");
//if (videoFrame.Direct3DSurface != null)
//{
// Debug.LogError(videoFrame.Direct3DSurface.Description.Format);
// Debug.LogError(videoFrame.Direct3DSurface.Description.Width);
// Debug.LogError(videoFrame.Direct3DSurface.Description.Height);
//}
}, false);
}
}
void Update()
{
if (textToSpeech.IsSpeaking())
{
RoboHeadAnimController.SetInteger("ChangeStateAnimation", 1);
}
else
{
RoboHeadAnimController.SetInteger("ChangeStateAnimation", 0);
}
}
void Update()
{
if (textToSpeech.IsSpeaking())
{
SceneAnimatorController.SetInteger("SpeakState", 2);
}
else
{
SceneAnimatorController.SetInteger("SpeakState", 0);
}
}
private IEnumerator WaitUntilSoundIsDone()
{
#if UNITY_EDITOR
int index = 0;
while (index < 5)
{
index++;
yield return(new WaitForSeconds(0.5f));
}
#else
yield return(new WaitWhile(() => TextToSpeech.SpeechTextInQueue() || TextToSpeech.IsSpeaking()));
#endif
Speek(_currentVoiceIndex++);
}
//Based on set language, function returns true/false if synthesizer is speaking
public bool IsSpeakingOrInQueue()
{
switch (language)
{
//use embeded text to speech
case Texts.Languages.ENGLISH:
return(textToSpeechDefault.IsSpeaking() || textToSpeechDefault.SpeechTextInQueue());
//use azure's text to speech cloud
case Texts.Languages.CZECH:
return(textToSpeechAzure.IsSpeaking());
default:
return(false);
}
}
// Update is called once per frame
void Update()
{
if (agent.enabled == true && !textToSpeech.IsSpeaking())
{
textToSpeech.StartSpeaking("Follow me");
if (Vector3.SqrMagnitude(Camera.main.transform.position - agent.transform.position) > 2.7)
{
agent.isStopped = true;
}
else
{
agent.isStopped = false;
}
if (Vector3.SqrMagnitude(agent.transform.position - localActiveDestination.transform.position) < 3)
{
agent.enabled = false;
textToSpeech.StopSpeaking();
}
}
}
// Update is called once per frame
private void Update()
{
// Update Display if exist
Update_DebugDisplay();
if (SpatialUnderstanding.Instance.ScanState == SpatialUnderstanding.ScanStates.Scanning &&
DoesScanMeetMinBarForCompletion && !speechTriggered)
{
if (!textToSpeech.IsSpeaking())
{
speechTriggered = true;
textToSpeech.StartSpeaking("Scan Finished. Click once more to procced");
}
}
if (!_triggered && SpatialUnderstanding.Instance.ScanState == SpatialUnderstanding.ScanStates.Done)
{
_triggered = true;
//Stop listening for clicks
EventManager.StopListening("click", TapListener);
//Trigger event for scan_done
EventManager.TriggerEvent("scan_done");
}
}
private void Update()
{
if (activeDestination != null)
{
if (Vector3.SqrMagnitude(Camera.main.transform.position - activeDestination.transform.position) < 2.7 && !textToSpeech.IsSpeaking())
{
textToSpeech.StartSpeaking("You have arrived.");
lr.enabled = false;
activeDestination = null;
}
}
}
void UpdatePropertiesDisplay()
{
// Update properties that change
if (qrCode != null && lastTimeStamp != qrCode.SystemRelativeLastDetectedTime.Ticks)
{
QRSize.text = "Size:" + qrCode.PhysicalSideLength.ToString("F04") + "m";
QRTimeStamp.text = "Time:" + qrCode.LastDetectedTime.ToString("MM/dd/yyyy HH:mm:ss.fff");
QRTimeStamp.color = QRTimeStamp.color == Color.yellow ? Color.white : Color.yellow;
PhysicalSize = qrCode.PhysicalSideLength;
Debug.Log("Id= " + qrCode.Id + "NodeId= " + qrCode.SpatialGraphNodeId + " PhysicalSize = " + PhysicalSize + " TimeStamp = " + qrCode.SystemRelativeLastDetectedTime.Ticks + " Time = " + qrCode.LastDetectedTime.ToString("MM/dd/yyyy HH:mm:ss.fff"));
qrCodeCube.transform.localPosition = new Vector3(PhysicalSize / 2.0f, PhysicalSize / 2.0f, 0.0f);
qrCodeCube.transform.localScale = new Vector3(PhysicalSize, PhysicalSize, 0.005f);
lastTimeStamp = qrCode.SystemRelativeLastDetectedTime.Ticks;
QRInfo.transform.localScale = new Vector3(PhysicalSize / 0.2f, PhysicalSize / 0.2f, PhysicalSize / 0.2f);
//AROA EDIT
QRPositionText.text = "Position: " + qrCodeCube.transform.position;
Debug.Log("Position = " + qrCodeCube.transform.position);
if (trackedObject != null)
{
if (trackedObject.name == "Collocated Cues") //Moving whole room
{
if (!textToSpeech.IsSpeaking())
{
textToSpeech.StartSpeaking(layout);
}
//obstacleManager.ResetPositions(); //Reset positions of obstacles
trackedObject.transform.eulerAngles = new Vector3(0f, 0f, 0f);
trackedObject.transform.Rotate(0f, qrCodeCube.transform.rotation.eulerAngles.y + 90f, 0f); //Rotate to match QR code, then 90 degrees
trackedObject.transform.position = qrCodeCube.transform.position; //new Vector3 (0.9f, 0f, 0f); //Adjust position to QR code location
//make local changes to adjust
//trackedObject.transform.localPosition -= trackedObject.transform.forward * 0.86f;
trackedObject.transform.localPosition -= trackedObject.transform.up * startingHeight; //Assumes height of QR Code is 1.5m (~5ft) off the ground
trackedObject.transform.localPosition += trackedObject.transform.right * startingDist; //Assumes hallway is 1.8m wide
highHeight = obstacleManager.highObstHeight;
if (layout == "Layout 1")
{
//Assign obstacles to position 1
obstWide1.transform.localPosition = new Vector3(-0.45f, wideHeight, 1.5f);
obstWide2.transform.localPosition = new Vector3(0.45f, wideHeight, 3.0f);
obstLow1.transform.localPosition = new Vector3(1000f, lowHeight, 1000f);
obstLow2.transform.localPosition = new Vector3(0f, lowHeight, 13.5f);
obstHigh1.transform.localPosition = new Vector3(0f, highHeight, 6.0f);
obstHigh2.transform.localPosition = new Vector3(0f, highHeight, 12.0f);
}
else if (layout == "Layout 2")
{
//Assign obstacles to position 2
obstWide1.transform.localPosition = new Vector3(-0.45f, wideHeight, 1.5f);
obstWide2.transform.localPosition = new Vector3(0.45f, wideHeight, 4.5f);
obstLow1.transform.localPosition = new Vector3(0f, lowHeight, 6f);
obstLow2.transform.localPosition = new Vector3(0f, lowHeight, 7.5f);
obstHigh1.transform.localPosition = new Vector3(0f, highHeight, 13.5f);
obstHigh2.transform.localPosition = new Vector3(0f, highHeight, 12.0f);
}
else if (layout == "Layout 3")
{
//Assign obstacles to position 3
obstWide1.transform.localPosition = new Vector3(-0.45f, wideHeight, 3.0f);
obstWide2.transform.localPosition = new Vector3(0.45f, wideHeight, 4.5f);
obstLow1.transform.localPosition = new Vector3(1000f, lowHeight, 1000f);
obstLow2.transform.localPosition = new Vector3(0f, lowHeight, 9.0f);
obstHigh1.transform.localPosition = new Vector3(0f, highHeight, 6.0f);
obstHigh2.transform.localPosition = new Vector3(0f, highHeight, 13.5f);
}
else if (layout == "Layout 4")
{
//Assign obstacles to position 4
obstWide1.transform.localPosition = new Vector3(-0.45f, wideHeight, 3.0f);
obstWide2.transform.localPosition = new Vector3(0.45f, wideHeight, 1.5f);
obstLow1.transform.localPosition = new Vector3(0f, lowHeight, 4.5f);
obstLow2.transform.localPosition = new Vector3(0f, lowHeight, 10.5f);
obstHigh1.transform.localPosition = new Vector3(0f, highHeight, 6.0f);
obstHigh2.transform.localPosition = new Vector3(0f, highHeight, 7.5f);
}
else if (layout == "Layout 5")
{
//Assign obstacles to position 5
obstWide1.transform.localPosition = new Vector3(-0.45f, wideHeight, 4.5f);
obstWide2.transform.localPosition = new Vector3(0.45f, wideHeight, 1.5f);
obstLow1.transform.localPosition = new Vector3(0f, lowHeight, 10.5f);
obstLow2.transform.localPosition = new Vector3(0f, lowHeight, 13.5f);
obstHigh1.transform.localPosition = new Vector3(0f, highHeight, 7.5f);
obstHigh2.transform.localPosition = new Vector3(0f, highHeight, 12.0f);
}
else if (layout == "Layout 6")
{
//Assign obstacles to position 6
obstWide1.transform.localPosition = new Vector3(-0.45f, wideHeight, 4.5f);
obstWide2.transform.localPosition = new Vector3(0.45f, wideHeight, 3.0f);
obstLow1.transform.localPosition = new Vector3(0f, lowHeight, 6f);
obstLow2.transform.localPosition = new Vector3(0f, lowHeight, 10.5f);
obstHigh1.transform.localPosition = new Vector3(0f, highHeight, 7.5f);
obstHigh2.transform.localPosition = new Vector3(1000f, highHeight, 1000f);
}
else if (layout == "Layout 7")
{
//Assign obstacles to position 7
obstWide1.transform.localPosition = new Vector3(-0.45f, wideHeight, 4.5f);
obstWide2.transform.localPosition = new Vector3(0.45f, wideHeight, 1.5f);
obstLow1.transform.localPosition = new Vector3(0f, lowHeight, 6f);
obstLow2.transform.localPosition = new Vector3(0f, lowHeight, 10.5f);
obstHigh1.transform.localPosition = new Vector3(0f, highHeight, 7.5f);
obstHigh2.transform.localPosition = new Vector3(0f, highHeight, 13.5f);
}
else if (layout == "Layout 8")
{
//Assign obstacles to position 8
obstWide1.transform.localPosition = new Vector3(-0.45f, wideHeight, 1.5f);
obstWide2.transform.localPosition = new Vector3(0.45f, wideHeight, 3.0f);
obstLow1.transform.localPosition = new Vector3(0f, lowHeight, 10.5f);
obstLow2.transform.localPosition = new Vector3(1000f, lowHeight, 1000f);
obstHigh1.transform.localPosition = new Vector3(0f, highHeight, 9f);
obstHigh2.transform.localPosition = new Vector3(0f, highHeight, 13.5f);
}
else if (layout == "Demo Layout")
{
//Assign a low obstacle to the middle of the room, and put the rest far away.
obstWide1.transform.localPosition = new Vector3(1000f, wideHeight, 1000f);
obstWide2.transform.localPosition = new Vector3(1000f, wideHeight, 1000f);
obstLow1.transform.localPosition = new Vector3(1000f, lowHeight, 1000f);
obstLow2.transform.localPosition = new Vector3(2f, lowHeight, 2f);
obstHigh1.transform.localPosition = new Vector3(1000f, highHeight, 1000f);
obstHigh2.transform.localPosition = new Vector3(1000f, highHeight, 1000f);
}
experimentLogger.layout = layout;
Debug.Log("Room position adjusted via QR code.");
}
else //Normal object
{
if (!textToSpeech.IsSpeaking())
{
textToSpeech.StartSpeaking("QR code detected.");
}
trackedObject.transform.localPosition = new Vector3(0f, 0f, 0f); //reset local position
trackedObject.transform.position = qrCodeCube.transform.position;
trackedObject.transform.rotation = qrCodeCube.transform.rotation;
}
}
}
}
