/* * increase the priority and weight of the grammar that start from the given index * and remove the priority and weight for the rest of the active grammars. */ public void ReloadAndChangeGrammarPriority(int index) { foreach (Grammar g in currentStoryGrammars) { if (g.RuleName.CompareTo("index_" + index) == 0) { loadGrammarCompleted = false; loadGrammarCount++; Debug.WriteLine("reloading grammar(index)..." + "\t" + EBookUtil.GetUnixTimeMillis() + "\t" + g.RuleName); recEngine.UnloadGrammar(g); g.Weight = EBookConstant.NEXT_WORD_WEIGHT; g.Priority = EBookConstant.NEXT_WORD_PRIORITY; recEngine.LoadGrammarAsync(g); } else if (g.Weight != EBookConstant.DEFAULT_WEIGHT || g.Priority != EBookConstant.DEFAULT_PRIORITY) { loadGrammarCompleted = false; loadGrammarCount++; Debug.WriteLine("reloading grammar(weight=+" + g.Weight + ",p=" + g.Priority + ")..." + "\t" + EBookUtil.GetUnixTimeMillis() + "\t" + g.RuleName); recEngine.UnloadGrammar(g); g.Weight = EBookConstant.DEFAULT_WEIGHT; g.Priority = EBookConstant.DEFAULT_PRIORITY; recEngine.LoadGrammarAsync(g); } } }
/* * Enable/disable SR * Disable SR when you absolutely don't need to recognize a speech. * You can keep the SR running when activate/deactivate grammars */ public void enableSR(bool b) { if (b) { if (!recOn) { while (!loadGrammarCompleted && loadGrammarCount != 0) { Thread.Sleep(3); } Debug.WriteLine("is load grammar complete before enable SR? " + loadGrammarCompleted + "\t" + EBookUtil.GetUnixTimeMillis()); ebookStream.enable(true); recEngine.RecognizeAsync(RecognizeMode.Multiple); recOn = true; Debug.WriteLine("Rec on"); } } else { if (recOn) { ebookStream.enable(false); recEngine.RecognizeAsyncCancel();//.RecognizeAsyncStop(); recOn = false; Debug.WriteLine("Rec off"); } } }
/// <summary> /// Log the "start of speech" /// </summary> /// <param name="state">the speech state</param> public void logSpeechState(SpeechState state) { if (state == SpeechState.SPEECH_START) { oneSpeech.Add("start::" + EBookUtil.GetUnixTimeMillis() + "\n"); } }
void recEngine_AudioLevelUpdated(object sender, AudioLevelUpdatedEventArgs e) { String timeStamp = GetTimestamp(DateTime.Now); Trace.WriteLine("audio level:" + e.AudioLevel + "\t" + timeStamp); if (e.AudioLevel > EBookInteractiveSystem.initialNoiseSensitivity) { int start = 0; double unixTime = EBookUtil.GetUnixTimeMillis(); if (preAudioLevel == 0) { //audio energy level jump start = 1; //audioStartTime = EBookUtil.GetUnixTimeMillis(); } //AbstractEBookEvent.raise(new AudioLevelChangeEvent(e.AudioLevel, start, unixTime)); ActivityExecutor.add(new InternalAudioLevelChangeActivity(e.AudioLevel, start, unixTime)); } else if (e.AudioLevel == 0 && preAudioLevel > 0 && e.AudioLevel == 0) { //audio energy level drop //AbstractEBookEvent.raise(new AudioLevelChangeEvent(e.AudioLevel, -1, 0)); ActivityExecutor.add(new InternalAudioLevelChangeActivity(e.AudioLevel, -1, 0)); } preAudioLevel = e.AudioLevel; }
/* * Load the given grammar. */ public void LoadOnGoingGrammar(Grammar g) { loadGrammarCompleted = false; loadGrammarCount++; Debug.WriteLine("loading onGoing grammar..." + "\t" + EBookUtil.GetUnixTimeMillis()); recEngine.LoadGrammarAsync(g); onGoingGrammars.Add(g); }
/* * Load the given story grammar. */ public void LoadStoryGrammar(Grammar g) { Debug.WriteLine("loading story grammar(" + g.Weight + "," + g.Priority + ")..." + g.RuleName + "\t" + EBookUtil.GetUnixTimeMillis()); loadGrammarCompleted = false; recEngine.LoadGrammarAsync(g); currentStoryGrammars.Add(g); }
/// <summary> /// The SR detects the audio energy jumps to a significant level, but hasn't yet declear /// it is a speech. The audio can be the beginning of a speech, or any other noise. In /// all cases, we want to start highlighting the next word to reduce the delay. /// </summary> /// <param name="audioStartTime">The start time of audio</param> private void processAudioEnergyHypothesizedHighlight(double audioStartTime) { if (EBookInteractiveSystem.initialLookAhead > 0) { double elapsedTime = EBookUtil.GetUnixTimeMillis() - audioStartTime; int syIn = (int)(elapsedTime / timePerSyllable); int MAX_INITIAL_HIGHLIGHT = EBookInteractiveSystem.initialLookAhead; int forLoopMax = confirmedEndIndex + 1 + MAX_INITIAL_HIGHLIGHT; int syCount = 0; for (int i = confirmedEndIndex + 1; i < forLoopMax && i < syllableArray.Length; i++) { syCount += syllableArray[i]; } //int syIn = syCount / EBookDialogueSystem.lookAheadDivider; //one syllable forward for every lookAheadDivide //make sure we can highlight at least one word if (confirmedEndIndex + 1 < syllableArray.Length && syIn < syllableArray[confirmedEndIndex + 1]) { syIn = syllableArray[confirmedEndIndex + 1]; } //the upperbound if (syIn > syCount) { syIn = syCount; } int enInc = confirmedEndIndex + 1; Boolean cont = true; int increment = 0; while (syIn > 0 && cont) { if (enInc < syllableArray.Length) { if (syIn >= syllableArray[enInc]) { increment++; } syIn -= syllableArray[enInc]; enInc++; } else { break; } } Trace.WriteLine("(audio level)guess " + increment + " word(s) ahead"); startIndex = confirmedStartIndex; endIndex = confirmedEndIndex + increment; } constructTextAndDisplay(); }
/* * Unload the active story grammar from SR */ public void UnloadStoryGrammar() { //stopSR(); if (currentStoryGrammars.Count > 0) { foreach (Grammar g in currentStoryGrammars) { recEngine.UnloadGrammar(g); Debug.WriteLine("unloading story grammar(" + g.Weight + "," + g.Priority + ")..." + g.RuleName + "\t" + EBookUtil.GetUnixTimeMillis()); } currentStoryGrammars.Clear(); } // startSR(); }
/// <summary> /// A new speech recognition received from SR. Save the result to a list. /// </summary> /// <param name="confidence">confident score</param> /// <param name="textResult">the text string of the speech</param> /// <param name="isHypothesis">is a hypothesis result?</param> /// <param name="semanticResult">the semantic that define in grammar </param> /// <param name="grammarName">the grammar that used for this speech</param> /// <param name="ruleName">The rule name that define in the grammar</param> /// <param name="audioDuration">The duration of the speech</param> /// <param name="wavPath">the path to the audio file</param> public void logRecognitionResult(float confidence, string textResult, bool isHypothesis, KeyValuePair <string, SemanticValue>[] semanticResult, string grammarName, string ruleName, double audioDuration, string wavPath) { double unixTime = EBookUtil.GetUnixTimeMillis(); string semantics = ""; foreach (KeyValuePair <string, SemanticValue> each in semanticResult) { semantics += "key::" + each.Key.ToString() + "|value::" + each.Value.Value.ToString() + "|"; } oneSpeech.Add("time::" + unixTime + "|confidence::" + confidence + "|textResult::" + textResult + "|isHypothesis::" + isHypothesis + "|" + semantics + "grammarName::" + grammarName + "|ruleName::" + ruleName + "|duration::" + audioDuration + "|wavePath::" + wavPath + "\n"); if (!isHypothesis) { lastCompleteSpeechIndex = oneSpeech.Count; } }
private void execThread() { while (on) { while (queue.Count == 0) { Thread.Sleep(IDLE_TIME); } Activity activity; if (queue.TryDequeue(out activity)) { AbstractSystemActivity sysActivity = null; if (activity.getActivityType() == ActivityType.USER_ACTIVITY || activity.getActivityType() == ActivityType.LIGHTWEIGHT_ACTIVITY) { sysActivity = activityModel.getSystemActivity(activity); } else if (activity.getActivityType() == ActivityType.SYSTEM_ACTIVITY) { sysActivity = (AbstractSystemActivity)activity; } Debug.WriteLine(EBookUtil.GetTimestamp() + " : " + activity + "->" + sysActivity); historyList.Add(activity); //TODO: run this in a separate thread? if (sysActivity != null) { sysActivity.execute(historyList); if (historyList.Count == HISTORY_MAX_SIZE) { historyList.RemoveAt(0); } historyList.Add(sysActivity); } } } }
/// <summary> /// Extract data out from page object. If enableCache is true, this method /// will save the data in cache for later fast access. /// </summary> /// <param name="page">The page object</param> private void retrivePageContent(Page page) { //retrieve page data from cache if (enableCache && pageTextCached.ContainsKey(page.GetPageNumber())) { pageTextCached.TryGetValue(page.GetPageNumber(), out allSpeechText); pageTextHTMLCached.TryGetValue(page.GetPageNumber(), out allSpeechTextHTML); annotationArrayCached.TryGetValue(page.GetPageNumber(), out annotationArray); syllableArrayCached.TryGetValue(page.GetPageNumber(), out syllableArray); } else { List <string[]> pageText = page.GetListOfTextArray(); List <string> annotation = page.GetListOfAnnotations(); List <string> annArray = new List <string>(); //cover list to array string whole = ""; string wholeHTML = ""; //obtain the text and the text in HTML format for the current page for (int i = 0; i < pageText.Count; i++) { if (pageText.ElementAt(i) == null) { wholeHTML = wholeHTML.TrimEnd() + "<br> "; } else { foreach (string str in pageText.ElementAt(i)) { whole += str + " "; if (str.Trim().Length == 0) { wholeHTML = wholeHTML.TrimEnd() + " "; } else { wholeHTML += str + " "; } annArray.Add(annotation.ElementAt(i)); } //wholeHTML = wholeHTML.TrimEnd() + "<br> "; } } whole = whole.Replace("\"", ""); allSpeechText = whole.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries); allSpeechTextHTML = wholeHTML.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries); annotationArray = annArray.ToArray(); syllableArray = EBookUtil.CountSyllables(allSpeechText); //save the data to hash map, we can simply retrieve the page data when //the page get revisit again if (enableCache) { pageTextCached.Add(page.GetPageNumber(), allSpeechText); pageTextHTMLCached.Add(page.GetPageNumber(), allSpeechTextHTML); annotationArrayCached.Add(page.GetPageNumber(), annotationArray); syllableArrayCached.Add(page.GetPageNumber(), syllableArray); } } }
void recEngine_LoadGrammarCompleted(object sender, LoadGrammarCompletedEventArgs e) { Debug.WriteLine("Load grammar completed" + "\t" + EBookUtil.GetUnixTimeMillis()); loadGrammarCount--; loadGrammarCompleted = true; }
public Activity() { creationTime = EBookUtil.GetUnixTimeMillis(); }
/* * return 1 if the result is a complete speech and it is the end of the page. */ private int processSpeechResult(string result, int startInd, bool isH, float confidence, double duration) { //lock this method, in case the this method will take more time than the SR lock (lockThis) { int ret = 0; string[] arr = result.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries); if (isH) { int increment = 0; if (guessAhead && EBookInteractiveSystem.lookAheadDivider > 0 && syllableArray.Length >= arr.Length + startInd) { //numOfHypothsis++; int syCount = 0; for (int i = startInd; i < arr.Length + startInd; i++) { syCount += syllableArray[i]; } int syIn = syCount / EBookInteractiveSystem.lookAheadDivider; //one syllable forward for every lookAheadDivide if (syIn < INITIAL_LOOK_AHEAD) { //The highlighting seems slow in the first second of a speech, let's highlight 2 syllables //ahead in the beginning of a speech syIn = INITIAL_LOOK_AHEAD; } int enInc = startInd + arr.Length; if (maxSyllableAhead > 0) { //Debug.WriteLine("Time pier syllable=" + timePerSyllable); if (syIn > maxSyllableAhead) { syIn = (int)maxSyllableAhead; } } string currentEndWord = ""; if (enInc > 0 && enInc <= allSpeechText.Length) { currentEndWord = allSpeechText[enInc - 1]; } Boolean cont = true; while (syIn > 0 && cont) { if (enInc < syllableArray.Length) { string guessWord = allSpeechText[enInc]; //if the current end word has pause punctuation, stop look ahead if (EBookUtil.containPausePunctuation(currentEndWord)) { Debug.WriteLine("currentEndWord \"" + currentEndWord + "\" contains pause, stop look ahead"); break; } else if (EBookUtil.containPausePunctuation(guessWord)) { //reduce 4 syllables from enInc syIn -= 3; //no guess ahead if there is any possible pause in guess ahead word. //Debug.WriteLine("guessWord \"" + guessWord + "\" contains pause, stop look ahead"); //cont = false; } if (syIn >= syllableArray[enInc]) { increment++; } syIn -= syllableArray[enInc]; enInc++; } else { break; } } Debug.WriteLine("guess " + increment + " word(s) ahead"); } //ONLY DISPLAY INTERMIDIATE RESULT WITH CONFIDENCE SCORE GREATER THAN HYP_THRESHOLD if (confidence > HYP_THRESHOLD) { if (startInd == lastSegPosition) //continue with last segment { startIndex = confirmedStartIndex; hypothesisEndIndex = arr.Length + lastSegPosition - 1; endIndex = hypothesisEndIndex + increment; } else if (arr.Length > 1) //jumping to onther sentence when at least two word in the speech { int gap = startInd - confirmedEndIndex; if (gap > 0 && gap <= SPEECH_JUMP_GAP_THRESHOLD) { //if reader skip maximum of SPEECH_JUMP_GAP_THRESHOLD, consider //the continue highlight the sentence } else { startIndex = startInd; } hypothesisEndIndex = arr.Length + startInd - 1; endIndex = hypothesisEndIndex + increment; } } } else { //duration != -1 only when !isH, so we need to estimate the time //per syllable for the next scentence using the previous speech int syCount = 0; for (int i = startInd; i < arr.Length + startInd; i++) { syCount += syllableArray[i]; } timePerSyllable = duration / syCount; Trace.WriteLine("timePerSyllable: " + timePerSyllable); maxSyllableAhead = (int)(maxLag / timePerSyllable); // numOfHypothsis = 0; if (startInd == lastSegPosition) { //if number of confirmed word in the complete speech is one word less than imcomplete speech, //we can treat the last imcomplete speech as complete int completeEndIndex = arr.Length + lastSegPosition - 1; if (endIndex - completeEndIndex != 1) { endIndex = completeEndIndex; } lastSegPosition = endIndex + 1; } else { int gap = startInd - lastSegPosition; if (gap > 0 && gap <= SPEECH_JUMP_GAP_THRESHOLD) { //if reader skip maximum of SPEECH_JUMP_GAP_THRESHOLD, consider //the continue highlight the sentence } else { startIndex = startInd; } endIndex = arr.Length + startInd - 1; lastSegPosition = endIndex + 1; } confirmedStartIndex = startIndex; confirmedEndIndex = endIndex; confirmedLastSegPosition = lastSegPosition; if (mode != Mode.REPLAY) { //EBookSRDevice rec = EBookSRDevice.GetInstance(); //reach the end of the page if (confirmedEndIndex == allSpeechText.Length - 1) { //stop SR when transitioning to the next page //rec.enableSR(false); //ret = 1; //AbstractEBookEvent.raise(new FinishPageEvent()); } else { string cfgPath = storyDirectory + "\\" + GRAMMAR_TMP_DIR + "\\" + currentPage.GetPageNumber() + "_" + (endIndex + 1) + ".cfg"; if (!defaultStartIndexes.Contains(endIndex + 1)) { //rec.enableSR(false); //rec.UnloadOnGoingGrammar(); Grammar g = new Grammar(cfgPath); g.Weight = EBookConstant.NEXT_WORD_WEIGHT; g.Priority = EBookConstant.NEXT_WORD_PRIORITY; //rec.LoadOnGoingGrammar(g); ActivityExecutor.add(new InternalReloadOnGoingGrammarActivity(g)); Debug.WriteLine("Load onGoing grammar " + cfgPath); //rec.GenerateAndLoadOnGoingGrammars(allSpeechText,endIndex+1); //rec.enableSR(true); } else { //the next sentence has higher priority //rec.ReloadAndChangeGrammarPriority(endIndex + 1); ActivityExecutor.add(new InternalChangeGrammarPriorityActivity(endIndex + 1)); } } } if (mode != Mode.RECORD && confirmedEndIndex == allSpeechText.Length - 1) { //the complete recognized result reaches the end of the page ret = 1; } //Detect speech and complete //AbstractEBookEvent.raise(new CompleteSpeechEvent()); } Debug.WriteLine("startIndex=" + startIndex); Debug.WriteLine("endIndex=" + endIndex); Debug.WriteLine("lastSegPosition=" + lastSegPosition); Debug.WriteLine("confirmedStartIndex=" + confirmedStartIndex); Debug.WriteLine("confirmedEndIndex=" + confirmedEndIndex); Debug.WriteLine("confirmedLastSegPosition=" + confirmedLastSegPosition); return(ret); } }
//String lastCompleteResult = ""; void recEngine_SpeechRecognized(object sender, SpeechRecognizedEventArgs e) { float confidence = e.Result.Confidence; string textResult = e.Result.Text; //string semantics = e.Result.Semantics.Value + ""; string grammarName = e.Result.Grammar.Name; string ruleName = e.Result.Grammar.RuleName; double audioDuration = -1; if (e.Result.Audio != null) { audioDuration = e.Result.Audio.Duration.TotalMilliseconds; } //string phonetic = ""; //foreach (RecognizedWordUnit unit in e.Result.Words) //{ // phonetic += unit.Pronunciation + " "; //} //Debug.WriteLine(textResult + "[" + phonetic.TrimEnd() + "]"); Debug.WriteLine("audio duration=" + audioDuration); KeyValuePair <string, SemanticValue>[] kvp = e.Result.Semantics.ToArray(); string fileP = null; string relPath = null; //only write audio file when given path is not null if (saveAudioPath != null) { string indexerStr = waveFileNameIndexer + ""; while (indexerStr.Length < 4) { indexerStr = "0" + indexerStr; } fileP = saveAudioPath + "\\" + indexerStr + ".wav"; relPath = EBookUtil.convertAudioToRelativePath(@fileP); } //AbstractEBookEvent.raise(new RecognitionResultEvent(confidence, textResult, false, // kvp,grammarName,ruleName,audioDuration,@fileP)); ActivityExecutor.add(new InternalSpeechRecognitionResultActivity(confidence, textResult, false, kvp, grammarName, ruleName, audioDuration, relPath)); //only write audio file when given path is not null if (fileP != null) { //write audio to file FileStream stream = new FileStream(fileP, FileMode.Create); e.Result.Audio.WriteToWaveStream(stream); stream.Flush(); stream.Close(); unconfirmSaveAudioList.Add(fileP); Trace.WriteLine("write to file " + fileP); waveFileNameIndexer++; } String timeStamp = GetTimestamp(DateTime.Now); string text = "\n" + e.Result.Confidence + "\t" + e.Result.Text + "(complete)\t\t" + e.Result.Semantics.Value + "\t" + e.Result.Grammar.Name + "\t" + timeStamp; Trace.WriteLine(text); }
/// <summary> /// There is some delay in SR processing. the method is to guesstimate how many word the reader /// had been reading ahead when the SR return the result. /// </summary> /// <param name="currentSpeedStartIndex"></param> /// <param name="textArray"></param> /// <returns></returns> private int getGuessAheadCount(int currentSpeedStartIndex, string[] textArray) { int increment = 0; if (guessAhead && EBookInteractiveSystem.lookAheadDivider > 0 && syllableArray.Length >= textArray.Length + currentSpeedStartIndex) { int syCount = 0; for (int i = currentSpeedStartIndex; i < textArray.Length + currentSpeedStartIndex; i++) { syCount += syllableArray[i]; } int syIn = syCount / EBookInteractiveSystem.lookAheadDivider; //one syllable forward for every lookAheadDivide if (syIn < INITIAL_LOOK_AHEAD) { //The highlighting seems slow in the first second of a speech, let's highlight 2 syllables //ahead in the beginning of a speech syIn = INITIAL_LOOK_AHEAD; } int enInc = currentSpeedStartIndex + textArray.Length; if (maxSyllableAhead > 0) { //Debug.WriteLine("Time pier syllable=" + timePerSyllable); if (syIn > maxSyllableAhead) { syIn = (int)maxSyllableAhead; } } string currentEndWord = ""; if (enInc > 0 && enInc <= allSpeechText.Length) { currentEndWord = allSpeechText[enInc - 1]; } Boolean cont = true; while (syIn > 0 && cont) { if (enInc < syllableArray.Length) { string guessWord = allSpeechText[enInc]; //if the current end word has pause punctuation, stop look ahead if (EBookUtil.containPausePunctuation(currentEndWord)) { Debug.WriteLine("currentEndWord \"" + currentEndWord + "\" contains pause, stop look ahead"); break; } else if (EBookUtil.containPausePunctuation(guessWord)) { //reduce 4 syllables from enInc syIn -= 3; //no guess ahead if there is any possible pause in guess ahead word. //Debug.WriteLine("guessWord \"" + guessWord + "\" contains pause, stop look ahead"); //cont = false; } if (syIn >= syllableArray[enInc]) { increment++; } syIn -= syllableArray[enInc]; enInc++; } else { break; } } Debug.WriteLine("guess " + increment + " word(s) ahead"); } return(increment); }
/// <summary> /// The method is to process a new page (or refresh a page). /// Generate necessary grammars for the current page. /// Generate text and display in browser. /// If Mode == REAL TIME, enable user to start from /jump to any sentence. otherwise, /// disable this feature. /// </summary> /// <param name="page">The page to be display in browser</param> /// <param name="mode">The story mode {read it now, record my voice}</param> public void process(Page page, Mode mode) { this.mode = mode; currentPage = page; //reset parameters when process a new page resetParameters(); retrivePageContent(page); if (mode != Mode.REPLAY) { List <SrgsDocument> srgsDocs = EBookUtil.GenerateGrammars(allSpeechText, annotationArray); Debug.WriteLine("generated " + srgsDocs.Count + " grammar files"); //load grammars List <Grammar> gs = new List <Grammar>(); //loop from srgsDocs.Count to 0 will give the earlier sentence the higher priority (if //two sentence contain the same word, Microsft choose the latest grammar that load into its engine). for (int i = srgsDocs.Count; --i >= 0;) { string cfgPath = storyDirectory + "\\" + GRAMMAR_TMP_DIR + "\\" + page.GetPageNumber() + "_" + i + ".cfg"; Directory.CreateDirectory(storyDirectory + "\\" + GRAMMAR_TMP_DIR); EBookUtil.CompileGrammarToFile(srgsDocs.ElementAt(i), cfgPath); //allow user to start from new sentence in read time mode if (mode == Mode.REALTIME) { if (i == 0 || (i > 0 && (EBookUtil.containEndOfSentencePunctuation(allSpeechText[i - 1])))) { defaultStartIndexes.Add(i); Debug.WriteLine("loading grammar:" + cfgPath); Grammar storyG = new Grammar(cfgPath); storyG.Weight = EBookConstant.DEFAULT_WEIGHT; storyG.Priority = EBookConstant.DEFAULT_PRIORITY; gs.Add(storyG); } } else //in recording mode, only allow the reader to start from where she/he left off { if (i == 0) { defaultStartIndexes.Add(i); Debug.WriteLine("loading grammar:" + cfgPath); Grammar storyG = new Grammar(cfgPath); storyG.Weight = EBookConstant.DEFAULT_WEIGHT; storyG.Priority = EBookConstant.DEFAULT_PRIORITY; ActivityExecutor.add(new InternalReloadOnGoingGrammarActivity(storyG)); } } } if (gs.Count > 0) { ActivityExecutor.add(new InternalReloadStoryGrammarActivity(gs)); } } string pageTextStr = constructPageText(); ActivityExecutor.add(new InternalUpdatePageTextActivity(pageTextStr, page.pageNumber)); }
public void generateSynthesisData(Story story) { saveData = true; foreach (InstalledVoice iv in synthesizer.GetInstalledVoices()) { string voiceName = iv.VoiceInfo.Name; Debug.WriteLine("installed voice :" + voiceName); synthesizer.SelectVoice(voiceName); string path = story.getFullPath() + "\\" + EBookInteractiveSystem.voice_dir + "\\" + voiceName.Replace(" ", "_"); Directory.CreateDirectory(path); int index = 0; Page p = story.GetFirstPage(); logResultFile = path + "\\" + StoryLoggingDevice.logFileName; if (File.Exists(logResultFile)) { Debug.WriteLine("skip gnerating synthesis data, data found in " + path); return; } startTime = (long)LogPlayer.GetUnixTimestamp(); while (p != null) { //reset startIndex when start a new page startIndex = 0; List <string[]> textArray = p.GetListOfTextArray(); string whole = ""; foreach (string[] text in textArray) { if (text != null) { foreach (string ea in text) { whole += ea + " "; } } } string[] separator = { "!", "?", "." }; string[] sp = whole.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries); List <string> sentences = new List <string>(); string tmpStr = ""; foreach (string ea in sp) { tmpStr += ea + " "; foreach (string punc in separator) { if (ea.Contains(punc)) { sentences.Add(tmpStr.TrimEnd()); tmpStr = ""; break; } } } foreach (string ea in sentences) { Debug.WriteLine("generating tts for:" + ea); string audioPath = path + "\\" + getPrefix(index) + ".wav"; synthesizer.SetOutputToWaveFile(@audioPath); PromptBuilder pb = new PromptBuilder(); synthesizer.Rate = synthesisSpeakRate; prompt = ""; writeToFile("start::" + startTime + "\n"); startTime += 10; synthesizer.Speak(ea); index++; startTime += (totalSentenceDuration); string data = "time::" + startTime + "|confidence::0.99|textResult::" + prompt.Trim() + "|isHypothesis::False|key::startIndex|value::" + startIndex + "|grammarName::|ruleName::index_" + startIndex + "|duration::" + totalSentenceDuration + "|wavePath::" + EBookUtil.convertAudioToRelativePath(@audioPath) + "\n"; writeToFile(data); string[] tmp = ea.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries); startIndex += tmp.Length; startTime += 1000;//one second pause } writeToFile("finishPage\n"); p = story.GetNextPage(); } } saveData = false; }
/* * If Mode == REAL TIME, enable jump to different sentence. otherwise, disable jump to different sentence */ public void process(Page page, Mode mode) { this.mode = mode; currentPage = page; //reset parameters when process a new page resetParameters(); //retrieve data from cache if (enableCache && pageTextCached.ContainsKey(page.GetPageNumber())) { pageTextCached.TryGetValue(page.GetPageNumber(), out allSpeechText); pageTextHTMLCached.TryGetValue(page.GetPageNumber(), out allSpeechTextHTML); annotationArrayCached.TryGetValue(page.GetPageNumber(), out annotationArray); syllableArrayCached.TryGetValue(page.GetPageNumber(), out syllableArray); } else { List <string[]> pageText = page.GetListOfTextArray(); List <string> annotation = page.GetListOfAnnotations(); List <string> annArray = new List <string>(); //cover list to array string whole = ""; string wholeHTML = ""; //obtain the text and the text in HTML format for the current page for (int i = 0; i < pageText.Count; i++) { if (pageText.ElementAt(i) == null) { wholeHTML = wholeHTML.TrimEnd() + "<br> "; } else { foreach (string str in pageText.ElementAt(i)) { whole += str + " "; if (str.Trim().Length == 0) { wholeHTML = wholeHTML.TrimEnd() + " "; } else { wholeHTML += str + " "; } annArray.Add(annotation.ElementAt(i)); } //wholeHTML = wholeHTML.TrimEnd() + "<br> "; } } whole = whole.Replace("\"", ""); allSpeechText = whole.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries); allSpeechTextHTML = wholeHTML.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries); annotationArray = annArray.ToArray(); syllableArray = EBookUtil.CountSyllables(allSpeechText); //save the data to hash map, we can simply retrieve the page data when //the page get revisit again if (enableCache) { pageTextCached.Add(page.GetPageNumber(), allSpeechText); pageTextHTMLCached.Add(page.GetPageNumber(), allSpeechTextHTML); annotationArrayCached.Add(page.GetPageNumber(), annotationArray); syllableArrayCached.Add(page.GetPageNumber(), syllableArray); } } if (mode != Mode.REPLAY) { defaultStartIndexes.Clear(); List <SrgsDocument> srgsDocs = EBookUtil.GenerateGrammars(allSpeechText, annotationArray); Debug.WriteLine("generated " + srgsDocs.Count + " grammar files"); //load grammars List <Grammar> gs = new List <Grammar>(); //loop from srgsDocs.Count to 0 will give the early sentence the priority. for (int i = srgsDocs.Count; --i >= 0;) { string cfgPath = storyDirectory + "\\" + GRAMMAR_TMP_DIR + "\\" + page.GetPageNumber() + "_" + i + ".cfg"; Directory.CreateDirectory(storyDirectory + "\\" + GRAMMAR_TMP_DIR); CompileGrammar(srgsDocs.ElementAt(i), cfgPath); if (mode == Mode.REALTIME) { if (i == 0 || (i > 0 && (allSpeechText[i - 1].Contains("?") || allSpeechText[i - 1].Contains(".") || allSpeechText[i - 1].Contains("!")))) { defaultStartIndexes.Add(i); Debug.WriteLine("loading grammar:" + cfgPath); Grammar storyG = new Grammar(cfgPath); storyG.Weight = EBookConstant.DEFAULT_WEIGHT; storyG.Priority = EBookConstant.DEFAULT_PRIORITY; gs.Add(storyG); } } else { if (i == 0) { defaultStartIndexes.Add(i); Debug.WriteLine("loading grammar:" + cfgPath); Grammar storyG = new Grammar(cfgPath); storyG.Weight = EBookConstant.DEFAULT_WEIGHT; storyG.Priority = EBookConstant.DEFAULT_PRIORITY; ActivityExecutor.add(new InternalReloadOnGoingGrammarActivity(storyG)); } } } if (gs.Count > 0) { ActivityExecutor.add(new InternalReloadStoryGrammarActivity(gs)); } } string pageTextStr = constructPageText(); ActivityExecutor.add(new InternalUpdatePageTextActivity(pageTextStr, page.pageNumber)); }