// BEGIN_RECORD
private void onBeginRecordMessage(Dictionary <string, object> args)
{
Logger.Log("onBeginRecordMessage");
int sentenceIndex = StorybookStateManager.GetState().evaluatingSentenceIndex;
this.taskQueue.Enqueue(this.recordAudioForCurrentSentence(sentenceIndex));
}
// Send a message representing storybook state to the controller, in a new thread.
// Doesn't need to return Action because it's only used as a timer elapsed handler.
private void sendStorybookState(object _, System.Timers.ElapsedEventArgs __)
{
Dictionary <string, object> publish = new Dictionary <string, object>();
publish.Add("topic", Constants.STORYBOOK_STATE_TOPIC);
publish.Add("op", "publish");
// Note that this is protected by a lock, so although ROS messages could
// send out of order, the information within them will be consistent.
// And if the sending rate isn't too high, the likelihood of out of order messages
// is low, and inconsequential for the controller anyway.
// TODO: should devise a better scheme to make sure states are sent in order.
// Can also use the sequence numbers provided in the header.
// Or use a lock in this class so that only one state message can be sent at a time.
Dictionary <string, object> data = StorybookStateManager.GetRosMessageData();
data.Add("header", RosbridgeUtilities.GetROSHeader());
// Don't allow audio_file to be null, ROS will get upset.
if (data["audio_file"] == null)
{
data["audio_file"] = "";
}
publish.Add("msg", data);
bool success = this.rosClient.SendMessage(Json.Serialize(publish));
if (!success)
{
// Logger.Log("Failed to send StorybookState message: " + Json.Serialize((publish)));
}
}
// Send a message representing storybook state to the controller.
// Doesn't need to return Action because it's only used as a timer elapsed handler.
private void sendStorybookState(object _, System.Timers.ElapsedEventArgs __)
{
Dictionary <string, object> publish = new Dictionary <string, object>();
publish.Add("topic", Constants.STORYBOOK_STATE_TOPIC);
publish.Add("op", "publish");
// TODO: could devise a better scheme to make sure states are sent in order.
// Can also use the sequence numbers provided in the header. Probably overkill.
Dictionary <string, object> data = StorybookStateManager.GetRosMessageData();
data.Add("header", RosbridgeUtilities.GetROSHeader());
// Don't allow audio_file to be null, ROS will get upset.
if (data["audio_file"] == null)
{
data["audio_file"] = "";
}
publish.Add("msg", data);
bool success = this.rosClient.SendMessage(Json.Serialize(publish));
if (!success)
{
// Logger.Log("Failed to send StorybookState message: " + Json.Serialize((publish)));
}
}
private Action showNextSentence(bool childTurn, bool shouldRecord)
{
return(() => {
Logger.Log("Showing next sentence from inside task queue");
if (StorybookStateManager.GetState().evaluatingSentenceIndex + 1 <
this.storyManager.stanzaManager.GetNumSentences())
{
StorybookStateManager.IncrementEvaluatingSentenceIndex();
int newIndex = StorybookStateManager.GetState().evaluatingSentenceIndex;
Color color = new Color();
if (childTurn)
{
color = Constants.CHILD_READ_TEXT_COLOR;
}
else
{
color = Constants.JIBO_READ_TEXT_COLOR;
}
this.storyManager.stanzaManager.GetSentence(newIndex).FadeIn(color);
if (newIndex - 1 >= 0)
{
this.storyManager.stanzaManager.GetSentence(newIndex - 1).ChangeTextColor(Constants.GREY_TEXT_COLOR);
}
if (shouldRecord)
{
Logger.Log("shouldRecord is true in showNextSentence");
this.recordAudioForCurrentSentence(newIndex).Invoke();
}
}
else
{
throw new Exception("Cannot show sentence, index out of range");
}
});
}
private Action setStorybookMode(int mode)
{
return(() => {
// Convert to StorybookMode.
StorybookMode newMode = (StorybookMode)mode;
StorybookStateManager.SetStorybookMode(newMode);
});
}
// Helper function to wrap together two actions:
// (1) loading a page and (2) sending the StorybookPageInfo message over ROS.
private void loadPageAndSendRosMessage(SceneDescription sceneDescription)
{
// Load the page.
this.storyManager.LoadPage(sceneDescription);
// Send the ROS message to update the controller about what page we're on now.
StorybookPageInfo updatedInfo = new StorybookPageInfo();
updatedInfo.storyName = this.currentStory.GetName();
updatedInfo.pageNumber = this.currentPageNumber;
updatedInfo.sentences = this.storyManager.stanzaManager.GetAllSentenceTexts();
// Update state (will get automatically sent to the controller.
StorybookStateManager.SetStorySelected(this.currentStory.GetName(),
this.currentStory.GetNumPages());
// Gather information about scene objects.
StorybookSceneObject[] sceneObjects =
new StorybookSceneObject[sceneDescription.sceneObjects.Length];
for (int i = 0; i < sceneDescription.sceneObjects.Length; i++)
{
SceneObject so = sceneDescription.sceneObjects[i];
StorybookSceneObject sso = new StorybookSceneObject();
sso.id = so.id;
sso.label = so.label;
sso.inText = so.inText;
sceneObjects[i] = sso;
}
updatedInfo.sceneObjects = sceneObjects;
// Gather information about tinker texts.
StorybookTinkerText[] tinkerTexts =
new StorybookTinkerText[this.storyManager.tinkerTexts.Count];
for (int i = 0; i < this.storyManager.tinkerTexts.Count; i++)
{
TinkerText tt = this.storyManager.tinkerTexts[i].GetComponent <TinkerText>();
StorybookTinkerText stt = new StorybookTinkerText();
stt.word = tt.word;
stt.hasSceneObject = false;
stt.sceneObjectId = -1;
tinkerTexts[i] = stt;
}
foreach (Trigger trigger in sceneDescription.triggers)
{
if (trigger.type == TriggerType.CLICK_TINKERTEXT_SCENE_OBJECT)
{
tinkerTexts[trigger.args.textId].hasSceneObject = true;
tinkerTexts[trigger.args.textId].sceneObjectId = trigger.args.sceneObjectId;
}
}
updatedInfo.tinkerTexts = tinkerTexts;
// Send the message.
if (Constants.USE_ROS)
{
this.rosManager.SendStorybookPageInfoAction(updatedInfo);
}
}
// Use Update for handling when to trigger actions in other objects.
private void Update()
{
if (this.audioSource.isPlaying)
{
// Check our current timestamp, and compare against timestamps of
// the triggers we have, in order to cause specific actions to happen.
this.currentTimestamp = this.audioSource.time;
float maxCutoffTime = this.currentTimestamp;
float minCutoffTime = Math.Max(this.lastTimestamp, this.startTimestamp);
// Watch for special case where the audio has finished and we need to
// make sure we call any outstanding triggers.
if (this.currentTimestamp < this.lastTimestamp)
{
maxCutoffTime = float.MaxValue;
}
foreach (KeyValuePair <float, List <AudioTrigger> > trigger in this.triggers)
{
// TODO: need a special case for first one? but not for hungry toad
if (trigger.Key >= minCutoffTime &&
trigger.Key <= maxCutoffTime)
{
// Invoke this trigger's action.
foreach (AudioTrigger t in trigger.Value)
{
if (!t.disallowInvokePastStop)
{
t.action();
}
else
{
// Only invoke if current time has not past stop time.
if (this.currentTimestamp <= this.stopTimestamp)
{
t.action();
}
else
{
Logger.Log("don't do trigger action because first in stanza");
}
}
}
}
}
if (this.currentTimestamp > this.stopTimestamp)
{
Logger.Log("stopping because current is " + this.currentTimestamp + " and stop is " + this.stopTimestamp);
this.StopAudio();
}
}
this.lastTimestamp = this.currentTimestamp;
// Update audio state so that StorybookState ROS messages are accurate.
bool playing = this.audioSource.isPlaying;
StorybookStateManager.SetAudioState(playing, this.audioFileName);
}
// Constructor.
public RosManager(string rosIP, string portNum, GameController gameController)
{
Logger.Log("RosManager constructor");
this.gameController = gameController;
this.storybookStateManager = gameController.GetStorybookStateManager();
this.rosClient = new RosbridgeWebSocketClient(rosIP, portNum);
this.rosClient.receivedMsgEvent += this.onMessageReceived;
this.commandHandlers = new Dictionary <StorybookCommand, Action <Dictionary <string, object> > >();
}
private void finishStory()
{
this.storyManager.ClearPage();
this.storyManager.audioManager.StopAudio();
this.storyManager.ShowTheEndPage(false);
this.currentPageNumber = 0;
this.setLandscapeOrientation();
this.showLibraryPanel(true);
StorybookStateManager.SetStoryExited();
}
void Start()
{
// Set up all UI elements. (SetActive, GetComponent, etc.)
// Get references to objects if necessary.
Logger.Log("Game Controller start");
this.landscapeNextButton.interactable = true;
this.landscapeNextButton.onClick.AddListener(this.onNextButtonClick);
this.portraitNextButton.interactable = true;
this.portraitNextButton.onClick.AddListener(this.onNextButtonClick);
this.landscapeBackButton.interactable = true;
this.landscapeBackButton.onClick.AddListener(this.onBackButtonClick);
this.portraitBackButton.interactable = true;
this.portraitBackButton.onClick.AddListener(this.onBackButtonClick);
this.landscapeFinishButton.interactable = true;
this.landscapeFinishButton.onClick.AddListener(this.onFinishButtonClick);
this.portraitFinishButton.interactable = true;
this.portraitFinishButton.onClick.AddListener(this.onFinishButtonClick);
this.rosConnectButton.onClick.AddListener(this.onRosConnectButtonClicked);
this.enterLibraryButton.onClick.AddListener(this.onEnterLibraryButtonClicked);
this.startStoryButton.onClick.AddListener(this.onStartStoryClicked);
this.landscapeToggleAudioButton.onClick.AddListener(this.toggleAudio);
this.portraitToggleAudioButton.onClick.AddListener(this.toggleAudio);
// Update the sizing of all of the panels depending on the actual
// screen size of the device we're on.
this.resizePanelsOnStartup();
this.storyPages = new List <SceneDescription>();
this.storybookStateManager = new StorybookStateManager();
this.storyManager = GetComponent <StoryManager>();
this.assetManager = GetComponent <AssetManager>();
this.audioRecorder = GetComponent <AudioRecorder>();
this.speechAceManager = GetComponent <SpeechAceManager>();
this.stories = new List <StoryMetadata>();
this.initStories();
// Either show the rosPanel to connect to ROS, or wait to go into story selection.
if (Constants.USE_ROS)
{
this.setupRosScreen();
}
// TODO: figure out when to actually set this, should be dependent on game mode.
this.storyManager.SetAutoplay(false);
}
// SHOW_NEXT_SENTENCE
private void onShowNextSentenceMessage(Dictionary <string, object> args)
{
// Assert that we are highlighting the appropriate sentence.
// Need to cast better.
Logger.Log("onShowNextSentenceMessage");
if (Convert.ToInt32(args["index"]) != StorybookStateManager.GetState().evaluatingSentenceIndex + 1)
{
Logger.LogError("Sentence index doesn't match " + args["index"] + " " +
StorybookStateManager.GetState().evaluatingSentenceIndex + 1);
throw new Exception("Sentence index doesn't match, fail fast");
}
this.taskQueue.Enqueue(this.showNextSentence((bool)args["child_turn"], (bool)args["record"]));
}
private void goToNextPage()
{
this.currentPageNumber += 1;
if (this.currentPageNumber > StorybookStateManager.GetState().numPages)
{
throw new Exception("Cannot go forward anymore, already at end " + this.currentPageNumber + " " + StorybookStateManager.GetState().numPages);
}
this.storyManager.ClearPage();
StorybookStateManager.ResetEvaluatingSentenceIndex();
// Explicitly send the state to make sure it gets sent before the page info does.
if (Constants.USE_ROS)
{
this.rosManager.SendStorybookState();
}
this.loadPageAndSendRosMessage(this.storyPages[this.currentPageNumber]);
}
private int currentPageNumber = 0; // 0-indexed, index into this.storyPages, 0 is title page.
void Awake()
{
// Enforce singleton pattern.
if (instance == null)
{
instance = this;
}
else if (instance != this)
{
Logger.Log("duplicate GameController, destroying");
Destroy(gameObject);
}
DontDestroyOnLoad(gameObject);
// Do this in Awake() to avoid null references.
StorybookStateManager.Init();
}
// =================
// Helpers.
// =================
// Separate the logic of showing buttons from actually moving pages.
// In evaluate mode, we want to be able to instruct the tablet to navigate the pages
// without the child needing to press any buttons.
private void goToPrevPage()
{
this.currentPageNumber -= 1;
if (this.currentPageNumber < 0)
{
// Fail fast.
throw new Exception("Cannot go back any farther, already at beginning");
}
this.storyManager.ClearPage();
StorybookStateManager.ResetEvaluatingSentenceIndex();
// Explicitly send the state to make sure it gets sent before the page info does.
if (Constants.USE_ROS)
{
this.rosManager.SendStorybookState();
}
this.loadPageAndSendRosMessage(this.storyPages[this.currentPageNumber]);
}
private void loadFirstPage()
{
if (Constants.USE_ROS)
{
this.rosManager.SendStorybookLoaded().Invoke();
}
this.loadPageAndSendRosMessage(this.storyPages[this.currentPageNumber]);
this.showLibraryPanel(false);
this.hideElement(this.loadingBar);
this.showElement(this.nextButton.gameObject);
this.showElement(this.toggleAudioButton.gameObject);
this.setOrientationView(this.orientation);
// If in evaluate mode, don't show any navigation buttons.
if (StorybookStateManager.GetState().storybookMode == StorybookMode.Evaluate)
{
this.showNavigationButtons(false);
}
}
public StorybookStateManager()
{
if (instance == null)
{
instance = this;
}
else
{
throw new Exception("Cannot attempt to create multiple StorybookStateManagers");
}
this.stateLock = new Object();
// Set default values for start of interaction.
this.currentState = new StorybookState {
audioPlaying = false,
audioFile = "",
storybookMode = StorybookMode.NotReading,
currentStory = "",
numPages = 0,
evaluatingStanzaIndex = -1,
};
}
// Upload an audio file to the collected child audio bucket in S3.
// Argument audioPath should be the same as what was passed into SaveAudioAtPath in AudioRecorder.
public void S3UploadChildAudio(string audioPath)
{
// Use a prefix that includes story, page number, first 2 words of stanza, and date.
string s3Path = DateTime.Now.ToString("yyyy-MM-dd") + "/" + Constants.PARTICIPANT_ID + "/" +
StorybookStateManager.GetState().currentStory + "/" +
StorybookStateManager.GetState().storybookMode + "/" +
DateTime.Now.ToString("HH:mm:ss") + "_" + audioPath;
PutObjectRequest request = new PutObjectRequest {
BucketName = Constants.S3_CHILD_AUDIO_BUCKET,
Key = s3Path,
FilePath = Application.persistentDataPath + "/" + audioPath
};
this.s3Client.PutObjectAsync(request, (responseObj) => {
if (responseObj.Exception == null)
{
Logger.Log("Successful upload " + s3Path);
}
else
{
Logger.Log("Upload failed");
}
});
}
// When child is done speaking, get SpeechACE results and save the recording.
private void stopRecordingAndDoSpeechace()
{
int sentenceIndex = StorybookStateManager.GetState().evaluatingSentenceIndex;
string text = this.storyManager.stanzaManager.GetSentence(sentenceIndex).GetSentenceText();
string tempFileName = this.currentPageNumber + "_" + sentenceIndex + ".wav";
this.audioRecorder.EndRecording((clip) => {
if (clip == null)
{
Logger.Log("Got null clip, means user pressed stop recording when no recording was active");
return;
}
Logger.Log("Done recording, getting speechACE results and uploading file to S3...");
// Tell controller we're done recording!
if (Constants.USE_ROS)
{
this.rosManager.SendRecordAudioComplete(sentenceIndex).Invoke();
}
// TODO: should also delete these audio files after we don't need them anymore.
AudioRecorder.SaveAudioAtPath(tempFileName, clip);
float duration = clip.length;
StartCoroutine(this.speechAceManager.AnalyzeTextSample(
tempFileName, text, (speechAceResult) => {
if (Constants.USE_ROS)
{
this.rosManager.SendSpeechAceResultAction(sentenceIndex, text,
duration, speechAceResult).Invoke();
}
// If we want to replay for debugging, uncomment this.
// AudioClip loadedClip = AudioRecorder.LoadAudioLocal(fileName);
// this.storyManager.audioManager.LoadAudio(loadedClip);
// this.storyManager.audioManager.PlayAudio();
this.assetManager.S3UploadChildAudio(tempFileName);
}));
});
}
public void SetStorybookStateManager(StorybookStateManager manager) {
this.storybookStateManager = manager;
}
// Main function to be called by GameController.
// Passes in a description received over ROS or hardcoded.
// LoadScene is responsible for loading all resources and putting them in
// place, and attaching callbacks to created GameObjects, where these
// callbacks involve functions from SceneManipulatorAPI.
public void LoadPage(SceneDescription description)
{
this.setDisplayModeFromSceneDescription(description);
this.resetPanelSizes();
// Only allow swiping in explore mode. TODO: maybe should change this if there's a need.
Stanza.ALLOW_SWIPE = StorybookStateManager.GetState().storybookMode == StorybookMode.Explore;
// Load audio.
this.audioManager.LoadAudio(description.audioFile, this.assetManager.GetAudioClip(description.audioFile));
if (description.isTitle)
{
// Show only the title panel.
this.titlePanel.SetActive(true);
this.readerPanel.SetActive(false);
// Special case for title page.
// No TinkerTexts, and image takes up a larger space.
this.loadTitlePage(description);
}
else
{
// Show only the text and graphics panels.
this.titlePanel.SetActive(false);
this.readerPanel.SetActive(true);
// Load image.
this.loadImage(description.storyImageFile);
List <string> textWords =
new List <string>(description.text.Split(' '));
// Need to remove any empty or only punctuation words.
textWords.RemoveAll(String.IsNullOrEmpty);
List <string> filteredTextWords = new List <string>();
foreach (string word in textWords)
{
if (Util.WordHasNoAlphanum(word))
{
filteredTextWords[filteredTextWords.Count - 1] += word;
}
else
{
filteredTextWords.Add(word);
}
}
if (filteredTextWords.Count != description.timestamps.Length)
{
Logger.LogError("textWords doesn't match timestamps length " +
filteredTextWords.Count + " " + description.timestamps.Length);
}
// Depending on how many words there are, update the sizing and spacing heuristically.
this.resizeSpacingAndFonts(filteredTextWords.Count);
for (int i = 0; i < filteredTextWords.Count; i++)
{
this.loadTinkerText(i, filteredTextWords[i], description.timestamps[i],
i == filteredTextWords.Count - 1);
}
// After all TinkerTexts and Stanzas have been formatted, set up all the sentences and
// set the stanza swipe handlers.
this.stanzaManager.SetupSentences();
// If we are in evaluate mode, all stanzas should be hidden by default.
if (StorybookStateManager.GetState().storybookMode == StorybookMode.Evaluate)
{
this.stanzaManager.HideAllSentences();
}
// This will send StorybookEvent ROS messages to the controller when stanzas are swiped.
if (Constants.USE_ROS)
{
this.stanzaManager.SetSentenceSwipeHandlers();
}
// Load audio triggers for TinkerText.
this.loadAudioTriggers();
}
// Load all scene objects.
foreach (SceneObject sceneObject in description.sceneObjects)
{
this.loadSceneObject(sceneObject);
}
// Sort scene objects by size (smallest on top).
this.sortSceneObjectLayering();
// Load triggers.
foreach (Trigger trigger in description.triggers)
{
this.loadTrigger(trigger);
}
if (this.autoplayAudio)
{
this.audioManager.PlayAudio();
}
}
