/*
* increase the priority and weight of the grammar that start from the given index
* and remove the priority and weight for the rest of the active grammars.
*/
public void ReloadAndChangeGrammarPriority(int index)
{
foreach (Grammar g in currentStoryGrammars)
{
if (g.RuleName.CompareTo("index_" + index) == 0)
{
loadGrammarCompleted = false;
loadGrammarCount++;
Debug.WriteLine("reloading grammar(index)..." + "\t" + EBookUtil.GetUnixTimeMillis() + "\t" +
g.RuleName);
recEngine.UnloadGrammar(g);
g.Weight = EBookConstant.NEXT_WORD_WEIGHT;
g.Priority = EBookConstant.NEXT_WORD_PRIORITY;
recEngine.LoadGrammarAsync(g);
}
else if (g.Weight != EBookConstant.DEFAULT_WEIGHT || g.Priority != EBookConstant.DEFAULT_PRIORITY)
{
loadGrammarCompleted = false;
loadGrammarCount++;
Debug.WriteLine("reloading grammar(weight=+" + g.Weight + ",p=" + g.Priority + ")..." +
"\t" + EBookUtil.GetUnixTimeMillis() + "\t" + g.RuleName);
recEngine.UnloadGrammar(g);
g.Weight = EBookConstant.DEFAULT_WEIGHT;
g.Priority = EBookConstant.DEFAULT_PRIORITY;
recEngine.LoadGrammarAsync(g);
}
}
}
/*
* Enable/disable SR
* Disable SR when you absolutely don't need to recognize a speech.
* You can keep the SR running when activate/deactivate grammars
*/
public void enableSR(bool b)
{
if (b)
{
if (!recOn)
{
while (!loadGrammarCompleted && loadGrammarCount != 0)
{
Thread.Sleep(3);
}
Debug.WriteLine("is load grammar complete before enable SR? " +
loadGrammarCompleted + "\t" + EBookUtil.GetUnixTimeMillis());
ebookStream.enable(true);
recEngine.RecognizeAsync(RecognizeMode.Multiple);
recOn = true;
Debug.WriteLine("Rec on");
}
}
else
{
if (recOn)
{
ebookStream.enable(false);
recEngine.RecognizeAsyncCancel();//.RecognizeAsyncStop();
recOn = false;
Debug.WriteLine("Rec off");
}
}
}
/// <summary>
/// Log the "start of speech"
/// </summary>
/// <param name="state">the speech state</param>
public void logSpeechState(SpeechState state)
{
if (state == SpeechState.SPEECH_START)
{
oneSpeech.Add("start::" + EBookUtil.GetUnixTimeMillis() + "\n");
}
}
void recEngine_AudioLevelUpdated(object sender, AudioLevelUpdatedEventArgs e)
{
String timeStamp = GetTimestamp(DateTime.Now);
Trace.WriteLine("audio level:" + e.AudioLevel + "\t" + timeStamp);
if (e.AudioLevel > EBookInteractiveSystem.initialNoiseSensitivity)
{
int start = 0;
double unixTime = EBookUtil.GetUnixTimeMillis();
if (preAudioLevel == 0)
{
//audio energy level jump
start = 1;
//audioStartTime = EBookUtil.GetUnixTimeMillis();
}
//AbstractEBookEvent.raise(new AudioLevelChangeEvent(e.AudioLevel, start, unixTime));
ActivityExecutor.add(new InternalAudioLevelChangeActivity(e.AudioLevel, start, unixTime));
}
else if (e.AudioLevel == 0 && preAudioLevel > 0 && e.AudioLevel == 0)
{
//audio energy level drop
//AbstractEBookEvent.raise(new AudioLevelChangeEvent(e.AudioLevel, -1, 0));
ActivityExecutor.add(new InternalAudioLevelChangeActivity(e.AudioLevel, -1, 0));
}
preAudioLevel = e.AudioLevel;
}
/*
* Load the given grammar.
*/
public void LoadOnGoingGrammar(Grammar g)
{
loadGrammarCompleted = false;
loadGrammarCount++;
Debug.WriteLine("loading onGoing grammar..." + "\t" + EBookUtil.GetUnixTimeMillis());
recEngine.LoadGrammarAsync(g);
onGoingGrammars.Add(g);
}
/*
* Load the given story grammar.
*/
public void LoadStoryGrammar(Grammar g)
{
Debug.WriteLine("loading story grammar(" + g.Weight + "," + g.Priority + ")..." +
g.RuleName + "\t" + EBookUtil.GetUnixTimeMillis());
loadGrammarCompleted = false;
recEngine.LoadGrammarAsync(g);
currentStoryGrammars.Add(g);
}
/// <summary>
/// The SR detects the audio energy jumps to a significant level, but hasn't yet declear
/// it is a speech. The audio can be the beginning of a speech, or any other noise. In
/// all cases, we want to start highlighting the next word to reduce the delay.
/// </summary>
/// <param name="audioStartTime">The start time of audio</param>
private void processAudioEnergyHypothesizedHighlight(double audioStartTime)
{
if (EBookInteractiveSystem.initialLookAhead > 0)
{
double elapsedTime = EBookUtil.GetUnixTimeMillis() - audioStartTime;
int syIn = (int)(elapsedTime / timePerSyllable);
int MAX_INITIAL_HIGHLIGHT = EBookInteractiveSystem.initialLookAhead;
int forLoopMax = confirmedEndIndex + 1 + MAX_INITIAL_HIGHLIGHT;
int syCount = 0;
for (int i = confirmedEndIndex + 1; i < forLoopMax && i < syllableArray.Length; i++)
{
syCount += syllableArray[i];
}
//int syIn = syCount / EBookDialogueSystem.lookAheadDivider; //one syllable forward for every lookAheadDivide
//make sure we can highlight at least one word
if (confirmedEndIndex + 1 < syllableArray.Length && syIn < syllableArray[confirmedEndIndex + 1])
{
syIn = syllableArray[confirmedEndIndex + 1];
}
//the upperbound
if (syIn > syCount)
{
syIn = syCount;
}
int enInc = confirmedEndIndex + 1;
Boolean cont = true;
int increment = 0;
while (syIn > 0 && cont)
{
if (enInc < syllableArray.Length)
{
if (syIn >= syllableArray[enInc])
{
increment++;
}
syIn -= syllableArray[enInc];
enInc++;
}
else
{
break;
}
}
Trace.WriteLine("(audio level)guess " + increment + " word(s) ahead");
startIndex = confirmedStartIndex;
endIndex = confirmedEndIndex + increment;
}
constructTextAndDisplay();
}
/*
* Unload the active story grammar from SR
*/
public void UnloadStoryGrammar()
{
//stopSR();
if (currentStoryGrammars.Count > 0)
{
foreach (Grammar g in currentStoryGrammars)
{
recEngine.UnloadGrammar(g);
Debug.WriteLine("unloading story grammar(" + g.Weight + "," + g.Priority +
")..." + g.RuleName + "\t" + EBookUtil.GetUnixTimeMillis());
}
currentStoryGrammars.Clear();
}
// startSR();
}
/// <summary>
/// A new speech recognition received from SR. Save the result to a list.
/// </summary>
/// <param name="confidence">confident score</param>
/// <param name="textResult">the text string of the speech</param>
/// <param name="isHypothesis">is a hypothesis result?</param>
/// <param name="semanticResult">the semantic that define in grammar </param>
/// <param name="grammarName">the grammar that used for this speech</param>
/// <param name="ruleName">The rule name that define in the grammar</param>
/// <param name="audioDuration">The duration of the speech</param>
/// <param name="wavPath">the path to the audio file</param>
public void logRecognitionResult(float confidence, string textResult,
bool isHypothesis, KeyValuePair <string, SemanticValue>[] semanticResult, string grammarName,
string ruleName, double audioDuration, string wavPath)
{
double unixTime = EBookUtil.GetUnixTimeMillis();
string semantics = "";
foreach (KeyValuePair <string, SemanticValue> each in semanticResult)
{
semantics += "key::" + each.Key.ToString() + "|value::" + each.Value.Value.ToString() + "|";
}
oneSpeech.Add("time::" + unixTime + "|confidence::" + confidence + "|textResult::" + textResult +
"|isHypothesis::" + isHypothesis + "|" + semantics + "grammarName::" + grammarName + "|ruleName::" +
ruleName + "|duration::" + audioDuration + "|wavePath::" + wavPath + "\n");
if (!isHypothesis)
{
lastCompleteSpeechIndex = oneSpeech.Count;
}
}
private void execThread()
{
while (on)
{
while (queue.Count == 0)
{
Thread.Sleep(IDLE_TIME);
}
Activity activity;
if (queue.TryDequeue(out activity))
{
AbstractSystemActivity sysActivity = null;
if (activity.getActivityType() == ActivityType.USER_ACTIVITY ||
activity.getActivityType() == ActivityType.LIGHTWEIGHT_ACTIVITY)
{
sysActivity = activityModel.getSystemActivity(activity);
}
else if (activity.getActivityType() == ActivityType.SYSTEM_ACTIVITY)
{
sysActivity = (AbstractSystemActivity)activity;
}
Debug.WriteLine(EBookUtil.GetTimestamp() + " : " + activity + "->" + sysActivity);
historyList.Add(activity);
//TODO: run this in a separate thread?
if (sysActivity != null)
{
sysActivity.execute(historyList);
if (historyList.Count == HISTORY_MAX_SIZE)
{
historyList.RemoveAt(0);
}
historyList.Add(sysActivity);
}
}
}
}
/// <summary>
/// Extract data out from page object. If enableCache is true, this method
/// will save the data in cache for later fast access.
/// </summary>
/// <param name="page">The page object</param>
private void retrivePageContent(Page page)
{
//retrieve page data from cache
if (enableCache && pageTextCached.ContainsKey(page.GetPageNumber()))
{
pageTextCached.TryGetValue(page.GetPageNumber(), out allSpeechText);
pageTextHTMLCached.TryGetValue(page.GetPageNumber(), out allSpeechTextHTML);
annotationArrayCached.TryGetValue(page.GetPageNumber(), out annotationArray);
syllableArrayCached.TryGetValue(page.GetPageNumber(), out syllableArray);
}
else
{
List <string[]> pageText = page.GetListOfTextArray();
List <string> annotation = page.GetListOfAnnotations();
List <string> annArray = new List <string>();
//cover list to array
string whole = "";
string wholeHTML = "";
//obtain the text and the text in HTML format for the current page
for (int i = 0; i < pageText.Count; i++)
{
if (pageText.ElementAt(i) == null)
{
wholeHTML = wholeHTML.TrimEnd() + "<br> ";
}
else
{
foreach (string str in pageText.ElementAt(i))
{
whole += str + " ";
if (str.Trim().Length == 0)
{
wholeHTML = wholeHTML.TrimEnd() + " ";
}
else
{
wholeHTML += str + " ";
}
annArray.Add(annotation.ElementAt(i));
}
//wholeHTML = wholeHTML.TrimEnd() + "<br> ";
}
}
whole = whole.Replace("\"", "");
allSpeechText = whole.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
allSpeechTextHTML = wholeHTML.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
annotationArray = annArray.ToArray();
syllableArray = EBookUtil.CountSyllables(allSpeechText);
//save the data to hash map, we can simply retrieve the page data when
//the page get revisit again
if (enableCache)
{
pageTextCached.Add(page.GetPageNumber(), allSpeechText);
pageTextHTMLCached.Add(page.GetPageNumber(), allSpeechTextHTML);
annotationArrayCached.Add(page.GetPageNumber(), annotationArray);
syllableArrayCached.Add(page.GetPageNumber(), syllableArray);
}
}
}
void recEngine_LoadGrammarCompleted(object sender, LoadGrammarCompletedEventArgs e)
{
Debug.WriteLine("Load grammar completed" + "\t" + EBookUtil.GetUnixTimeMillis());
loadGrammarCount--;
loadGrammarCompleted = true;
}
public Activity()
{
creationTime = EBookUtil.GetUnixTimeMillis();
}
/*
* return 1 if the result is a complete speech and it is the end of the page.
*/
private int processSpeechResult(string result, int startInd, bool isH, float confidence, double duration)
{
//lock this method, in case the this method will take more time than the SR
lock (lockThis)
{
int ret = 0;
string[] arr = result.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
if (isH)
{
int increment = 0;
if (guessAhead && EBookInteractiveSystem.lookAheadDivider > 0 && syllableArray.Length >= arr.Length + startInd)
{
//numOfHypothsis++;
int syCount = 0;
for (int i = startInd; i < arr.Length + startInd; i++)
{
syCount += syllableArray[i];
}
int syIn = syCount / EBookInteractiveSystem.lookAheadDivider; //one syllable forward for every lookAheadDivide
if (syIn < INITIAL_LOOK_AHEAD)
{
//The highlighting seems slow in the first second of a speech, let's highlight 2 syllables
//ahead in the beginning of a speech
syIn = INITIAL_LOOK_AHEAD;
}
int enInc = startInd + arr.Length;
if (maxSyllableAhead > 0)
{
//Debug.WriteLine("Time pier syllable=" + timePerSyllable);
if (syIn > maxSyllableAhead)
{
syIn = (int)maxSyllableAhead;
}
}
string currentEndWord = "";
if (enInc > 0 && enInc <= allSpeechText.Length)
{
currentEndWord = allSpeechText[enInc - 1];
}
Boolean cont = true;
while (syIn > 0 && cont)
{
if (enInc < syllableArray.Length)
{
string guessWord = allSpeechText[enInc];
//if the current end word has pause punctuation, stop look ahead
if (EBookUtil.containPausePunctuation(currentEndWord))
{
Debug.WriteLine("currentEndWord \"" + currentEndWord + "\" contains pause, stop look ahead");
break;
}
else if (EBookUtil.containPausePunctuation(guessWord))
{
//reduce 4 syllables from enInc
syIn -= 3;
//no guess ahead if there is any possible pause in guess ahead word.
//Debug.WriteLine("guessWord \"" + guessWord + "\" contains pause, stop look ahead");
//cont = false;
}
if (syIn >= syllableArray[enInc])
{
increment++;
}
syIn -= syllableArray[enInc];
enInc++;
}
else
{
break;
}
}
Debug.WriteLine("guess " + increment + " word(s) ahead");
}
//ONLY DISPLAY INTERMIDIATE RESULT WITH CONFIDENCE SCORE GREATER THAN HYP_THRESHOLD
if (confidence > HYP_THRESHOLD)
{
if (startInd == lastSegPosition) //continue with last segment
{
startIndex = confirmedStartIndex;
hypothesisEndIndex = arr.Length + lastSegPosition - 1;
endIndex = hypothesisEndIndex + increment;
}
else if (arr.Length > 1) //jumping to onther sentence when at least two word in the speech
{
int gap = startInd - confirmedEndIndex;
if (gap > 0 && gap <= SPEECH_JUMP_GAP_THRESHOLD)
{
//if reader skip maximum of SPEECH_JUMP_GAP_THRESHOLD, consider
//the continue highlight the sentence
}
else
{
startIndex = startInd;
}
hypothesisEndIndex = arr.Length + startInd - 1;
endIndex = hypothesisEndIndex + increment;
}
}
}
else
{
//duration != -1 only when !isH, so we need to estimate the time
//per syllable for the next scentence using the previous speech
int syCount = 0;
for (int i = startInd; i < arr.Length + startInd; i++)
{
syCount += syllableArray[i];
}
timePerSyllable = duration / syCount;
Trace.WriteLine("timePerSyllable: " + timePerSyllable);
maxSyllableAhead = (int)(maxLag / timePerSyllable); //
numOfHypothsis = 0;
if (startInd == lastSegPosition)
{
//if number of confirmed word in the complete speech is one word less than imcomplete speech,
//we can treat the last imcomplete speech as complete
int completeEndIndex = arr.Length + lastSegPosition - 1;
if (endIndex - completeEndIndex != 1)
{
endIndex = completeEndIndex;
}
lastSegPosition = endIndex + 1;
}
else
{
int gap = startInd - lastSegPosition;
if (gap > 0 && gap <= SPEECH_JUMP_GAP_THRESHOLD)
{
//if reader skip maximum of SPEECH_JUMP_GAP_THRESHOLD, consider
//the continue highlight the sentence
}
else
{
startIndex = startInd;
}
endIndex = arr.Length + startInd - 1;
lastSegPosition = endIndex + 1;
}
confirmedStartIndex = startIndex;
confirmedEndIndex = endIndex;
confirmedLastSegPosition = lastSegPosition;
if (mode != Mode.REPLAY)
{
//EBookSRDevice rec = EBookSRDevice.GetInstance();
//reach the end of the page
if (confirmedEndIndex == allSpeechText.Length - 1)
{
//stop SR when transitioning to the next page
//rec.enableSR(false);
//ret = 1;
//AbstractEBookEvent.raise(new FinishPageEvent());
}
else
{
string cfgPath = storyDirectory + "\\" + GRAMMAR_TMP_DIR + "\\" +
currentPage.GetPageNumber() + "_" + (endIndex + 1) + ".cfg";
if (!defaultStartIndexes.Contains(endIndex + 1))
{
//rec.enableSR(false);
//rec.UnloadOnGoingGrammar();
Grammar g = new Grammar(cfgPath);
g.Weight = EBookConstant.NEXT_WORD_WEIGHT;
g.Priority = EBookConstant.NEXT_WORD_PRIORITY;
//rec.LoadOnGoingGrammar(g);
ActivityExecutor.add(new InternalReloadOnGoingGrammarActivity(g));
Debug.WriteLine("Load onGoing grammar " + cfgPath);
//rec.GenerateAndLoadOnGoingGrammars(allSpeechText,endIndex+1);
//rec.enableSR(true);
}
else
{
//the next sentence has higher priority
//rec.ReloadAndChangeGrammarPriority(endIndex + 1);
ActivityExecutor.add(new InternalChangeGrammarPriorityActivity(endIndex + 1));
}
}
}
if (mode != Mode.RECORD && confirmedEndIndex == allSpeechText.Length - 1)
{
//the complete recognized result reaches the end of the page
ret = 1;
}
//Detect speech and complete
//AbstractEBookEvent.raise(new CompleteSpeechEvent());
}
Debug.WriteLine("startIndex=" + startIndex);
Debug.WriteLine("endIndex=" + endIndex);
Debug.WriteLine("lastSegPosition=" + lastSegPosition);
Debug.WriteLine("confirmedStartIndex=" + confirmedStartIndex);
Debug.WriteLine("confirmedEndIndex=" + confirmedEndIndex);
Debug.WriteLine("confirmedLastSegPosition=" + confirmedLastSegPosition);
return(ret);
}
}
//String lastCompleteResult = "";
void recEngine_SpeechRecognized(object sender, SpeechRecognizedEventArgs e)
{
float confidence = e.Result.Confidence;
string textResult = e.Result.Text;
//string semantics = e.Result.Semantics.Value + "";
string grammarName = e.Result.Grammar.Name;
string ruleName = e.Result.Grammar.RuleName;
double audioDuration = -1;
if (e.Result.Audio != null)
{
audioDuration = e.Result.Audio.Duration.TotalMilliseconds;
}
//string phonetic = "";
//foreach (RecognizedWordUnit unit in e.Result.Words)
//{
// phonetic += unit.Pronunciation + " ";
//}
//Debug.WriteLine(textResult + "[" + phonetic.TrimEnd() + "]");
Debug.WriteLine("audio duration=" + audioDuration);
KeyValuePair <string, SemanticValue>[] kvp = e.Result.Semantics.ToArray();
string fileP = null;
string relPath = null;
//only write audio file when given path is not null
if (saveAudioPath != null)
{
string indexerStr = waveFileNameIndexer + "";
while (indexerStr.Length < 4)
{
indexerStr = "0" + indexerStr;
}
fileP = saveAudioPath + "\\" + indexerStr + ".wav";
relPath = EBookUtil.convertAudioToRelativePath(@fileP);
}
//AbstractEBookEvent.raise(new RecognitionResultEvent(confidence, textResult, false,
// kvp,grammarName,ruleName,audioDuration,@fileP));
ActivityExecutor.add(new InternalSpeechRecognitionResultActivity(confidence, textResult, false,
kvp, grammarName, ruleName, audioDuration, relPath));
//only write audio file when given path is not null
if (fileP != null)
{
//write audio to file
FileStream stream = new FileStream(fileP, FileMode.Create);
e.Result.Audio.WriteToWaveStream(stream);
stream.Flush();
stream.Close();
unconfirmSaveAudioList.Add(fileP);
Trace.WriteLine("write to file " + fileP);
waveFileNameIndexer++;
}
String timeStamp = GetTimestamp(DateTime.Now);
string text = "\n" + e.Result.Confidence + "\t" + e.Result.Text + "(complete)\t\t" +
e.Result.Semantics.Value + "\t" + e.Result.Grammar.Name + "\t" + timeStamp;
Trace.WriteLine(text);
}
/// <summary>
/// There is some delay in SR processing. the method is to guesstimate how many word the reader
/// had been reading ahead when the SR return the result.
/// </summary>
/// <param name="currentSpeedStartIndex"></param>
/// <param name="textArray"></param>
/// <returns></returns>
private int getGuessAheadCount(int currentSpeedStartIndex, string[] textArray)
{
int increment = 0;
if (guessAhead && EBookInteractiveSystem.lookAheadDivider > 0 && syllableArray.Length >= textArray.Length + currentSpeedStartIndex)
{
int syCount = 0;
for (int i = currentSpeedStartIndex; i < textArray.Length + currentSpeedStartIndex; i++)
{
syCount += syllableArray[i];
}
int syIn = syCount / EBookInteractiveSystem.lookAheadDivider; //one syllable forward for every lookAheadDivide
if (syIn < INITIAL_LOOK_AHEAD)
{
//The highlighting seems slow in the first second of a speech, let's highlight 2 syllables
//ahead in the beginning of a speech
syIn = INITIAL_LOOK_AHEAD;
}
int enInc = currentSpeedStartIndex + textArray.Length;
if (maxSyllableAhead > 0)
{
//Debug.WriteLine("Time pier syllable=" + timePerSyllable);
if (syIn > maxSyllableAhead)
{
syIn = (int)maxSyllableAhead;
}
}
string currentEndWord = "";
if (enInc > 0 && enInc <= allSpeechText.Length)
{
currentEndWord = allSpeechText[enInc - 1];
}
Boolean cont = true;
while (syIn > 0 && cont)
{
if (enInc < syllableArray.Length)
{
string guessWord = allSpeechText[enInc];
//if the current end word has pause punctuation, stop look ahead
if (EBookUtil.containPausePunctuation(currentEndWord))
{
Debug.WriteLine("currentEndWord \"" + currentEndWord + "\" contains pause, stop look ahead");
break;
}
else if (EBookUtil.containPausePunctuation(guessWord))
{
//reduce 4 syllables from enInc
syIn -= 3;
//no guess ahead if there is any possible pause in guess ahead word.
//Debug.WriteLine("guessWord \"" + guessWord + "\" contains pause, stop look ahead");
//cont = false;
}
if (syIn >= syllableArray[enInc])
{
increment++;
}
syIn -= syllableArray[enInc];
enInc++;
}
else
{
break;
}
}
Debug.WriteLine("guess " + increment + " word(s) ahead");
}
return(increment);
}
/// <summary>
/// The method is to process a new page (or refresh a page).
/// Generate necessary grammars for the current page.
/// Generate text and display in browser.
/// If Mode == REAL TIME, enable user to start from /jump to any sentence. otherwise,
/// disable this feature.
/// </summary>
/// <param name="page">The page to be display in browser</param>
/// <param name="mode">The story mode {read it now, record my voice}</param>
public void process(Page page, Mode mode)
{
this.mode = mode;
currentPage = page;
//reset parameters when process a new page
resetParameters();
retrivePageContent(page);
if (mode != Mode.REPLAY)
{
List <SrgsDocument> srgsDocs = EBookUtil.GenerateGrammars(allSpeechText, annotationArray);
Debug.WriteLine("generated " + srgsDocs.Count + " grammar files");
//load grammars
List <Grammar> gs = new List <Grammar>();
//loop from srgsDocs.Count to 0 will give the earlier sentence the higher priority (if
//two sentence contain the same word, Microsft choose the latest grammar that load into its engine).
for (int i = srgsDocs.Count; --i >= 0;)
{
string cfgPath = storyDirectory + "\\" + GRAMMAR_TMP_DIR + "\\" + page.GetPageNumber() + "_" + i + ".cfg";
Directory.CreateDirectory(storyDirectory + "\\" + GRAMMAR_TMP_DIR);
EBookUtil.CompileGrammarToFile(srgsDocs.ElementAt(i), cfgPath);
//allow user to start from new sentence in read time mode
if (mode == Mode.REALTIME)
{
if (i == 0 || (i > 0 && (EBookUtil.containEndOfSentencePunctuation(allSpeechText[i - 1]))))
{
defaultStartIndexes.Add(i);
Debug.WriteLine("loading grammar:" + cfgPath);
Grammar storyG = new Grammar(cfgPath);
storyG.Weight = EBookConstant.DEFAULT_WEIGHT;
storyG.Priority = EBookConstant.DEFAULT_PRIORITY;
gs.Add(storyG);
}
}
else //in recording mode, only allow the reader to start from where she/he left off
{
if (i == 0)
{
defaultStartIndexes.Add(i);
Debug.WriteLine("loading grammar:" + cfgPath);
Grammar storyG = new Grammar(cfgPath);
storyG.Weight = EBookConstant.DEFAULT_WEIGHT;
storyG.Priority = EBookConstant.DEFAULT_PRIORITY;
ActivityExecutor.add(new InternalReloadOnGoingGrammarActivity(storyG));
}
}
}
if (gs.Count > 0)
{
ActivityExecutor.add(new InternalReloadStoryGrammarActivity(gs));
}
}
string pageTextStr = constructPageText();
ActivityExecutor.add(new InternalUpdatePageTextActivity(pageTextStr, page.pageNumber));
}
public void generateSynthesisData(Story story)
{
saveData = true;
foreach (InstalledVoice iv in synthesizer.GetInstalledVoices())
{
string voiceName = iv.VoiceInfo.Name;
Debug.WriteLine("installed voice :" + voiceName);
synthesizer.SelectVoice(voiceName);
string path = story.getFullPath() + "\\" + EBookInteractiveSystem.voice_dir + "\\" + voiceName.Replace(" ", "_");
Directory.CreateDirectory(path);
int index = 0;
Page p = story.GetFirstPage();
logResultFile = path + "\\" + StoryLoggingDevice.logFileName;
if (File.Exists(logResultFile))
{
Debug.WriteLine("skip gnerating synthesis data, data found in " + path);
return;
}
startTime = (long)LogPlayer.GetUnixTimestamp();
while (p != null)
{
//reset startIndex when start a new page
startIndex = 0;
List <string[]> textArray = p.GetListOfTextArray();
string whole = "";
foreach (string[] text in textArray)
{
if (text != null)
{
foreach (string ea in text)
{
whole += ea + " ";
}
}
}
string[] separator = { "!", "?", "." };
string[] sp = whole.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
List <string> sentences = new List <string>();
string tmpStr = "";
foreach (string ea in sp)
{
tmpStr += ea + " ";
foreach (string punc in separator)
{
if (ea.Contains(punc))
{
sentences.Add(tmpStr.TrimEnd());
tmpStr = "";
break;
}
}
}
foreach (string ea in sentences)
{
Debug.WriteLine("generating tts for:" + ea);
string audioPath = path + "\\" + getPrefix(index) + ".wav";
synthesizer.SetOutputToWaveFile(@audioPath);
PromptBuilder pb = new PromptBuilder();
synthesizer.Rate = synthesisSpeakRate;
prompt = "";
writeToFile("start::" + startTime + "\n");
startTime += 10;
synthesizer.Speak(ea);
index++;
startTime += (totalSentenceDuration);
string data = "time::" + startTime + "|confidence::0.99|textResult::" + prompt.Trim() +
"|isHypothesis::False|key::startIndex|value::" + startIndex + "|grammarName::|ruleName::index_" +
startIndex + "|duration::" + totalSentenceDuration + "|wavePath::" + EBookUtil.convertAudioToRelativePath(@audioPath) + "\n";
writeToFile(data);
string[] tmp = ea.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
startIndex += tmp.Length;
startTime += 1000;//one second pause
}
writeToFile("finishPage\n");
p = story.GetNextPage();
}
}
saveData = false;
}
/*
* If Mode == REAL TIME, enable jump to different sentence. otherwise, disable jump to different sentence
*/
public void process(Page page, Mode mode)
{
this.mode = mode;
currentPage = page;
//reset parameters when process a new page
resetParameters();
//retrieve data from cache
if (enableCache && pageTextCached.ContainsKey(page.GetPageNumber()))
{
pageTextCached.TryGetValue(page.GetPageNumber(), out allSpeechText);
pageTextHTMLCached.TryGetValue(page.GetPageNumber(), out allSpeechTextHTML);
annotationArrayCached.TryGetValue(page.GetPageNumber(), out annotationArray);
syllableArrayCached.TryGetValue(page.GetPageNumber(), out syllableArray);
}
else
{
List <string[]> pageText = page.GetListOfTextArray();
List <string> annotation = page.GetListOfAnnotations();
List <string> annArray = new List <string>();
//cover list to array
string whole = "";
string wholeHTML = "";
//obtain the text and the text in HTML format for the current page
for (int i = 0; i < pageText.Count; i++)
{
if (pageText.ElementAt(i) == null)
{
wholeHTML = wholeHTML.TrimEnd() + "<br> ";
}
else
{
foreach (string str in pageText.ElementAt(i))
{
whole += str + " ";
if (str.Trim().Length == 0)
{
wholeHTML = wholeHTML.TrimEnd() + " ";
}
else
{
wholeHTML += str + " ";
}
annArray.Add(annotation.ElementAt(i));
}
//wholeHTML = wholeHTML.TrimEnd() + "<br> ";
}
}
whole = whole.Replace("\"", "");
allSpeechText = whole.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
allSpeechTextHTML = wholeHTML.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
annotationArray = annArray.ToArray();
syllableArray = EBookUtil.CountSyllables(allSpeechText);
//save the data to hash map, we can simply retrieve the page data when
//the page get revisit again
if (enableCache)
{
pageTextCached.Add(page.GetPageNumber(), allSpeechText);
pageTextHTMLCached.Add(page.GetPageNumber(), allSpeechTextHTML);
annotationArrayCached.Add(page.GetPageNumber(), annotationArray);
syllableArrayCached.Add(page.GetPageNumber(), syllableArray);
}
}
if (mode != Mode.REPLAY)
{
defaultStartIndexes.Clear();
List <SrgsDocument> srgsDocs = EBookUtil.GenerateGrammars(allSpeechText, annotationArray);
Debug.WriteLine("generated " + srgsDocs.Count + " grammar files");
//load grammars
List <Grammar> gs = new List <Grammar>();
//loop from srgsDocs.Count to 0 will give the early sentence the priority.
for (int i = srgsDocs.Count; --i >= 0;)
{
string cfgPath = storyDirectory + "\\" + GRAMMAR_TMP_DIR + "\\" + page.GetPageNumber() + "_" + i + ".cfg";
Directory.CreateDirectory(storyDirectory + "\\" + GRAMMAR_TMP_DIR);
CompileGrammar(srgsDocs.ElementAt(i), cfgPath);
if (mode == Mode.REALTIME)
{
if (i == 0 || (i > 0 && (allSpeechText[i - 1].Contains("?") || allSpeechText[i - 1].Contains(".") ||
allSpeechText[i - 1].Contains("!"))))
{
defaultStartIndexes.Add(i);
Debug.WriteLine("loading grammar:" + cfgPath);
Grammar storyG = new Grammar(cfgPath);
storyG.Weight = EBookConstant.DEFAULT_WEIGHT;
storyG.Priority = EBookConstant.DEFAULT_PRIORITY;
gs.Add(storyG);
}
}
else
{
if (i == 0)
{
defaultStartIndexes.Add(i);
Debug.WriteLine("loading grammar:" + cfgPath);
Grammar storyG = new Grammar(cfgPath);
storyG.Weight = EBookConstant.DEFAULT_WEIGHT;
storyG.Priority = EBookConstant.DEFAULT_PRIORITY;
ActivityExecutor.add(new InternalReloadOnGoingGrammarActivity(storyG));
}
}
}
if (gs.Count > 0)
{
ActivityExecutor.add(new InternalReloadStoryGrammarActivity(gs));
}
}
string pageTextStr = constructPageText();
ActivityExecutor.add(new InternalUpdatePageTextActivity(pageTextStr, page.pageNumber));
}
