/// <summary>
/// SR rejects the recent hypothesis result, roll the highlighting back to previous
/// comfirmed text
/// </summary>
public void rollBackText()
{
numOfHypothsis = 0;
lastSegPosition = confirmedLastSegPosition;
startIndex = confirmedStartIndex;
endIndex = confirmedEndIndex;
processAnimation(endIndex);
//construct the text without highlight if the rejected recognition is the first sentence/word of the page
if (confirmedLastSegPosition == confirmedStartIndex && confirmedStartIndex == confirmedEndIndex &&
confirmedEndIndex == 0)
{
string pageTextStr = constructPageText();
ActivityExecutor.add(new InternalUpdatePageTextActivity(pageTextStr, currentPage.GetPageNumber()));
}
else
{
//construct html text with highlight
constructTextAndDisplay();
}
}
/// <summary>
/// Interpret the meaning for a command speech.
/// Caution: the string comparison in this function for the commands are
/// rely on the script in command.grxml. If you can something in the command.grxml file
/// you may need to edit command string in this function, vice versa.
/// </summary>
/// <param name="semantics"></param>
private void processCommandSemantics(KeyValuePair <string, SemanticValue>[] semantics)
{
foreach (KeyValuePair <string, SemanticValue> each in semantics)
{
if (each.Key.CompareTo("NavigatePage") == 0)
{
if (each.Value.Value.ToString().CompareTo("[next]") == 0)
{
ActivityExecutor.add(new InternalSpeechNavigatePageActivity(PageAction.NEXT));
}
else if (each.Value.Value.ToString().CompareTo("[previous]") == 0)
{
ActivityExecutor.add(new InternalSpeechNavigatePageActivity(PageAction.PREVIOUS));
}
else
{
int pageN = Convert.ToInt32(each.Value.Value);
ActivityExecutor.add(new InternalSpeechNavigatePageActivity(PageAction.GO_PAGE_X, pageN));
}
}
}
}
public void playSyncAudio()
{
if (run)
{
Thread.Sleep(500);
if (threax != null && threax.IsAlive)
{
threax.Join();
}
Debug.WriteLine("##start replay audio [" + currentAudioIndex + "]");
if (currentAudioIndex < audioPlayList.Count && run)
{
string audioPath = audioPlayList.ElementAt(currentAudioIndex);
int pas = currentAudioIndex;
threax = new Thread(() => regenerateSREvent(pas));
threax.Start();
ActivityExecutor.add(new InternalReplayAudioActivity(audioPath, currentAudioIndex));
currentAudioIndex++;
}
}
}
void recEngine_SpeechHypothesized(object sender, SpeechHypothesizedEventArgs e)
{
float confidence = e.Result.Confidence;
string textResult = e.Result.Text;
string grammarName = e.Result.Grammar.Name;
string ruleName = e.Result.Grammar.RuleName;
double audioDuration = -1;
if (e.Result.Audio != null)
{
//Looks like we can't get the audio duration for hypothesis result, BUG in Microsoft?
audioDuration = e.Result.Audio.Duration.TotalMilliseconds;
}
KeyValuePair <string, SemanticValue>[] kvp = e.Result.Semantics.ToArray();
//AbstractEBookEvent.raise(new RecognitionResultEvent(confidence,textResult,true,
// kvp,grammarName,ruleName, audioDuration,null));
ActivityExecutor.add(new InternalSpeechRecognitionResultActivity(confidence, textResult, true,
kvp, grammarName, ruleName, audioDuration, null));
String timeStamp = GetTimestamp(DateTime.Now);
string text = "\n" + e.Result.Confidence + "\t" + e.Result.Text + "(Hypothesis)\t" +
e.Result.Semantics.Value + "\tgrammarName=" + e.Result.Grammar.Name + "\truleName=" +
e.Result.Grammar.RuleName + "\t" + timeStamp;
Trace.WriteLine(text);
//double timeNow = EBookUtil.GetUnixTimeMillis();
//double mun = timeNow - speechTurnAroundTime;
//speechTurnAroundTime = timeNow;
//eachSpeechTimes.Add(mun);
}
//String lastCompleteResult = "";
void recEngine_SpeechRecognized(object sender, SpeechRecognizedEventArgs e)
{
float confidence = e.Result.Confidence;
string textResult = e.Result.Text;
//string semantics = e.Result.Semantics.Value + "";
string grammarName = e.Result.Grammar.Name;
string ruleName = e.Result.Grammar.RuleName;
double audioDuration = -1;
if (e.Result.Audio != null)
{
audioDuration = e.Result.Audio.Duration.TotalMilliseconds;
}
//string phonetic = "";
//foreach (RecognizedWordUnit unit in e.Result.Words)
//{
// phonetic += unit.Pronunciation + " ";
//}
//Debug.WriteLine(textResult + "[" + phonetic.TrimEnd() + "]");
Debug.WriteLine("audio duration=" + audioDuration);
KeyValuePair <string, SemanticValue>[] kvp = e.Result.Semantics.ToArray();
string fileP = null;
string relPath = null;
//only write audio file when given path is not null
if (saveAudioPath != null)
{
string indexerStr = waveFileNameIndexer + "";
while (indexerStr.Length < 4)
{
indexerStr = "0" + indexerStr;
}
fileP = saveAudioPath + "\\" + indexerStr + ".wav";
relPath = EBookUtil.convertAudioToRelativePath(@fileP);
}
//AbstractEBookEvent.raise(new RecognitionResultEvent(confidence, textResult, false,
// kvp,grammarName,ruleName,audioDuration,@fileP));
ActivityExecutor.add(new InternalSpeechRecognitionResultActivity(confidence, textResult, false,
kvp, grammarName, ruleName, audioDuration, relPath));
//only write audio file when given path is not null
if (fileP != null)
{
//write audio to file
FileStream stream = new FileStream(fileP, FileMode.Create);
e.Result.Audio.WriteToWaveStream(stream);
stream.Flush();
stream.Close();
unconfirmSaveAudioList.Add(fileP);
Trace.WriteLine("write to file " + fileP);
waveFileNameIndexer++;
}
String timeStamp = GetTimestamp(DateTime.Now);
string text = "\n" + e.Result.Confidence + "\t" + e.Result.Text + "(complete)\t\t" +
e.Result.Semantics.Value + "\t" + e.Result.Grammar.Name + "\t" + timeStamp;
Trace.WriteLine(text);
}
/// <summary>
/// Restart the system
/// </summary>
public void restart()
{
ActivityExecutor.add(new UserRestartSystemActivity());
}
/// <summary>
/// process the recognition result, the recognition result can be the
/// hypothesis result or the complete result.
/// </summary>
/// <param name="confidence"></param>
/// <param name="textResult"></param>
/// <param name="isHypothesis"></param>
/// <param name="semantics"></param>
/// <param name="grammarName"></param>
/// <param name="ruleName"></param>
/// <param name="audioDuration"></param>
/// <param name="wavPath"></param>
public void processRecognitionResult(float confidence, string textResult,
bool isHypothesis, KeyValuePair <string, SemanticValue>[] semantics, string grammarName,
string ruleName, double audioDuration, string wavPath)
{
//handle result if the recognized speech is a command
if (grammarName.CompareTo("command") == 0)
{
if (!isHypothesis && confidence * 100 > EBookInteractiveSystem.commandConfidenceThreshold)
{
processCommandSemantics(semantics);
}
}
//handle result if this is story text
else
{
int start = -1; //the index of the first word of the recognized text in the current page
//process the story annotations
if (semantics != null && semantics.Length > 0)
{
foreach (KeyValuePair <string, SemanticValue> each in semantics)
{
if (each.Key.CompareTo("startIndex") == 0)
{
start = Convert.ToInt32(each.Value.Value);
}
}
}
if (start == -1)
{
string rule = ruleName;
if (rule.StartsWith("index_"))
{
string startIndex = rule.Substring(6);
if (startIndex.Length > 0)
{
start = Convert.ToInt32(startIndex);
}
}
}
//process the highlighting before animation (try to underline the text as fast as possible)
Debug.WriteLine(textResult);
Trace.WriteLine("start process Text time: " + DateTime.Now.ToString("yyyyMMddHHmmssfff"));
int isEndOfPage = processSpeechResult(textResult, start, isHypothesis,
confidence, audioDuration);
constructTextAndDisplay();
Trace.WriteLine("end process Text time: " + DateTime.Now.ToString("yyyyMMddHHmmssfff"));
//for some reasons, the hypothesis results do not contain any semantic results, so
//we have to find it manually
if (isHypothesis)
{
string[] tmp = textResult.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
//it is offen misrecognize the first word in a speech.
//generate animation when hypothsis text is greater than 1
if (tmp.Length > 1)
{
processHypothesisAnimation(tmp.Length, start);
}
}
else
{
string[] tmp = textResult.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
processAnimation(tmp.Length + start);
//keep the last action in the list, remove rest of the them
if (isEndOfPage == 1)
{
//Debug.WriteLine("generating a finishpageactivity "+isHypothesis);
ActivityExecutor.add(new InternalFinishPageActivity());
}
}
}
}
/// <summary>
/// construct the highlight text and fire up a internal activity
/// </summary>
private void constructTextAndDisplay()
{
//if startIndex not equals to confirmedStartIndex, the hypothesis result
//starts from different position.
string displayText = "<span class='storytext'>";
for (int i = 0; i < allSpeechTextHTML.Length; i++)
{
//the on going speech is not yet confirmed
if (startIndex == confirmedStartIndex && endIndex > confirmedEndIndex && startIndex != -1)
{
//if it is the beginning (no confirmed speech)
if (confirmedEndIndex == 0)
{
if (startIndex == i && endIndex == i)
{
if (endIndex > hypothesisEndIndex)
{
displayText += "<span class='hypothesis'>" + allSpeechTextHTML[i] + " </span> <span class='lookAhead'>";
}
else
{
displayText += "<span class='hypothesis'>" + allSpeechTextHTML[i] + " </span> ";
}
}
else if (startIndex == i)
{
displayText += " <span class='hypothesis'>" + allSpeechTextHTML[i] + " ";
}
else if (hypothesisEndIndex == i && hypothesisEndIndex < endIndex)
{
displayText += allSpeechTextHTML[i] + " </span> <span class='lookAhead'>";
}
else if (endIndex == i)
{
displayText += allSpeechTextHTML[i] + " </span> ";
}
else
{
displayText += allSpeechTextHTML[i] + " ";
}
}
else
{
if (startIndex == i && endIndex == i)
{
if (endIndex > hypothesisEndIndex)
{
displayText += "<span class='hypothesis'>" + allSpeechTextHTML[i] + " </span> <span class='lookAhead'>";
}
else
{
displayText += "<span class='hypothesis'>" + allSpeechTextHTML[i] + " </span> ";
}
}
else if (startIndex == i)
{
displayText += "<span class='highlight'>" + allSpeechTextHTML[i] + " ";
}
else if (confirmedEndIndex == i)
{
displayText += allSpeechTextHTML[i] + " </span> <span class='hypothesis'>";
}
else if (hypothesisEndIndex == i && hypothesisEndIndex < endIndex)
{
displayText += allSpeechTextHTML[i] + " </span> <span class='lookAhead'>";
}
else if (endIndex == i)
{
displayText += allSpeechTextHTML[i] + " </span> ";
}
else
{
displayText += allSpeechTextHTML[i] + " ";
}
}
}
//the on going speech is not yet confirmed and the hypothesis result starts from different position
else if (startIndex != confirmedStartIndex && startIndex != -1)
{
//display the confimred words
if (confirmedStartIndex == i && confirmedEndIndex == i)
{
displayText += "<span class='highlight'>" + allSpeechTextHTML[i] + " </span> ";
}
else if (confirmedStartIndex == i)
{
displayText += "<span class='highlight'>" + allSpeechTextHTML[i] + " ";
}
else if (confirmedEndIndex == i)
{
displayText += allSpeechTextHTML[i] + " </span> ";
}
else if (startIndex == i)
{
displayText += "<span class='hypothesis'>" + allSpeechTextHTML[i] + " ";
}
else if (endIndex == i)
{
displayText += allSpeechTextHTML[i] + " </span> ";
}
else
{
displayText += allSpeechTextHTML[i] + " ";
}
}
//all spoken speech are confirmed
else
{
if (startIndex == i && endIndex == i)
{
displayText += "<span class='highlight'>" + allSpeechTextHTML[i] + "</span> ";
}
else if (startIndex == i)
{
displayText += "<span class='highlight'>" + allSpeechTextHTML[i] + " ";
}
else if (endIndex == i)
{
displayText += allSpeechTextHTML[i] + "</span> ";
}
else
{
displayText += allSpeechTextHTML[i] + " ";
}
}
}
displayText += "</span>";
Debug.WriteLine(displayText);
ActivityExecutor.add(new InternalUpdateTextHighLightActivity(displayText));
}
/// <summary>
/// Process the result from speech recognizer. This method will figure out the
/// highlighting, and generate the grammar that start from the last word it left off.
/// </summary>
/// <param name="result">The recognized text</param>
/// <param name="currentSpeedStartIndex"></param>
/// <param name="isH">indicate whether the result is a hypothesis</param>
/// <param name="confidence">The confidence score between [0,1]</param>
/// <param name="duration">the duration of the detected speech in millisecond</param>
/// <returns>return 1 if the result is a complete speech and it is in the end of the page.</returns>
private int processSpeechResult(string result, int currentSpeedStartIndex, bool isH, float confidence, double duration)
{
//lock this method, in case the this method will take more time than the next result from SR
lock (lockThis)
{
int ret = 0;
string[] arr = result.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
if (isH)
{
int increment = getGuessAheadCount(currentSpeedStartIndex, arr);
//ONLY DISPLAY INTERMIDIATE RESULT WITH CONFIDENCE SCORE GREATER THAN HYP_THRESHOLD
if (confidence > HYP_THRESHOLD)
{
if (currentSpeedStartIndex == lastSegPosition + 1) //continue with last segment
{
if (confirmedStartIndex != -1)
{
startIndex = confirmedStartIndex;
}
else
{
startIndex = currentSpeedStartIndex;
}
hypothesisEndIndex = arr.Length + lastSegPosition;
endIndex = hypothesisEndIndex + increment;
}
else if (arr.Length > 1) //jumping to onther sentence when at least two word in the speech
{
int gap = 0;
if (confirmedEndIndex != -1)
{
if (currentSpeedStartIndex < confirmedEndIndex)
{
//the reader jump to the text prior to the last confirmed text
gap = SPEECH_JUMP_GAP_THRESHOLD + 1;
}
else
{
gap = currentSpeedStartIndex - confirmedEndIndex;
}
}
else
{
gap = currentSpeedStartIndex;
}
if (gap > 0 && gap <= SPEECH_JUMP_GAP_THRESHOLD)
{
//if reader skip maximum of SPEECH_JUMP_GAP_THRESHOLD, consider
//the continue highlight the sentence
}
else
{
startIndex = currentSpeedStartIndex;
}
hypothesisEndIndex = arr.Length + currentSpeedStartIndex - 1;
endIndex = hypothesisEndIndex + increment;
}
}
}
else
{
//duration != -1 only when !isH, so we need to estimate the time
//per syllable for the next scentence using the previous speech
int syCount = 0;
for (int i = currentSpeedStartIndex; i < arr.Length + currentSpeedStartIndex; i++)
{
syCount += syllableArray[i];
}
timePerSyllable = duration / syCount;
Trace.WriteLine("timePerSyllable: " + timePerSyllable);
maxSyllableAhead = (int)(maxLag / timePerSyllable); //
numOfHypothsis = 0;
if (currentSpeedStartIndex == lastSegPosition + 1)
{
//if number of confirmed word in the complete speech is one word less than hypothesis speech,
//we can treat the last imcomplete speech as complete
int completeEndIndex = arr.Length + lastSegPosition;
if (endIndex - completeEndIndex == 1)
{
//the completed speed is one word less than the previous hypothesis speech
}
else
{
endIndex = completeEndIndex;
}
lastSegPosition = endIndex;
}
//the reader 'skip' the sentence
else
{
int gap = currentSpeedStartIndex - lastSegPosition + 1;
if (currentSpeedStartIndex < lastSegPosition)
{
gap = SPEECH_JUMP_GAP_THRESHOLD + 1;
}
if (gap > 0 && gap <= SPEECH_JUMP_GAP_THRESHOLD)
{
//if reader skip maximum of SPEECH_JUMP_GAP_THRESHOLD, consider
//the continue highlight the sentence
}
else
{
startIndex = currentSpeedStartIndex;
}
endIndex = arr.Length + currentSpeedStartIndex - 1;
lastSegPosition = endIndex;
}
confirmedStartIndex = startIndex;
confirmedEndIndex = endIndex;
confirmedLastSegPosition = lastSegPosition;
if (mode != Mode.REPLAY)
{
//reach the end of the page
if (confirmedEndIndex == allSpeechText.Length - 1)
{
//do nothing. We don't need to handle the end of the page for
//replay mode in this class
}
else
{
string cfgPath = storyDirectory + "\\" + GRAMMAR_TMP_DIR + "\\" +
currentPage.GetPageNumber() + "_" + (endIndex + 1) + ".cfg";
if (!defaultStartIndexes.Contains(endIndex + 1))
{
Grammar g = new Grammar(cfgPath);
g.Weight = EBookConstant.NEXT_WORD_WEIGHT;
g.Priority = EBookConstant.NEXT_WORD_PRIORITY;
ActivityExecutor.add(new InternalReloadOnGoingGrammarActivity(g));
Debug.WriteLine("Load onGoing grammar " + cfgPath);
}
else
{
//the next sentence has higher priority
ActivityExecutor.add(new InternalChangeGrammarPriorityActivity(endIndex + 1));
}
}
}
if (mode != Mode.RECORD && confirmedEndIndex == allSpeechText.Length - 1)
{
//the complete recognized result reaches the end of the page
ret = 1;
}
}
Debug.WriteLine("startIndex=" + startIndex);
Debug.WriteLine("endIndex=" + endIndex);
Debug.WriteLine("lastSegPosition=" + lastSegPosition);
Debug.WriteLine("confirmedStartIndex=" + confirmedStartIndex);
Debug.WriteLine("confirmedEndIndex=" + confirmedEndIndex);
Debug.WriteLine("confirmedLastSegPosition=" + confirmedLastSegPosition);
return(ret);
}
}
/// <summary>
/// The method is to process a new page (or refresh a page).
/// Generate necessary grammars for the current page.
/// Generate text and display in browser.
/// If Mode == REAL TIME, enable user to start from /jump to any sentence. otherwise,
/// disable this feature.
/// </summary>
/// <param name="page">The page to be display in browser</param>
/// <param name="mode">The story mode {read it now, record my voice}</param>
public void process(Page page, Mode mode)
{
this.mode = mode;
currentPage = page;
//reset parameters when process a new page
resetParameters();
retrivePageContent(page);
if (mode != Mode.REPLAY)
{
List <SrgsDocument> srgsDocs = EBookUtil.GenerateGrammars(allSpeechText, annotationArray);
Debug.WriteLine("generated " + srgsDocs.Count + " grammar files");
//load grammars
List <Grammar> gs = new List <Grammar>();
//loop from srgsDocs.Count to 0 will give the earlier sentence the higher priority (if
//two sentence contain the same word, Microsft choose the latest grammar that load into its engine).
for (int i = srgsDocs.Count; --i >= 0;)
{
string cfgPath = storyDirectory + "\\" + GRAMMAR_TMP_DIR + "\\" + page.GetPageNumber() + "_" + i + ".cfg";
Directory.CreateDirectory(storyDirectory + "\\" + GRAMMAR_TMP_DIR);
EBookUtil.CompileGrammarToFile(srgsDocs.ElementAt(i), cfgPath);
//allow user to start from new sentence in read time mode
if (mode == Mode.REALTIME)
{
if (i == 0 || (i > 0 && (EBookUtil.containEndOfSentencePunctuation(allSpeechText[i - 1]))))
{
defaultStartIndexes.Add(i);
Debug.WriteLine("loading grammar:" + cfgPath);
Grammar storyG = new Grammar(cfgPath);
storyG.Weight = EBookConstant.DEFAULT_WEIGHT;
storyG.Priority = EBookConstant.DEFAULT_PRIORITY;
gs.Add(storyG);
}
}
else //in recording mode, only allow the reader to start from where she/he left off
{
if (i == 0)
{
defaultStartIndexes.Add(i);
Debug.WriteLine("loading grammar:" + cfgPath);
Grammar storyG = new Grammar(cfgPath);
storyG.Weight = EBookConstant.DEFAULT_WEIGHT;
storyG.Priority = EBookConstant.DEFAULT_PRIORITY;
ActivityExecutor.add(new InternalReloadOnGoingGrammarActivity(storyG));
}
}
}
if (gs.Count > 0)
{
ActivityExecutor.add(new InternalReloadStoryGrammarActivity(gs));
}
}
string pageTextStr = constructPageText();
ActivityExecutor.add(new InternalUpdatePageTextActivity(pageTextStr, page.pageNumber));
}
/*
* return 1 if the result is a complete speech and it is the end of the page.
*/
private int processSpeechResult(string result, int startInd, bool isH, float confidence, double duration)
{
//lock this method, in case the this method will take more time than the SR
lock (lockThis)
{
int ret = 0;
string[] arr = result.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
if (isH)
{
int increment = 0;
if (guessAhead && EBookInteractiveSystem.lookAheadDivider > 0 && syllableArray.Length >= arr.Length + startInd)
{
//numOfHypothsis++;
int syCount = 0;
for (int i = startInd; i < arr.Length + startInd; i++)
{
syCount += syllableArray[i];
}
int syIn = syCount / EBookInteractiveSystem.lookAheadDivider; //one syllable forward for every lookAheadDivide
if (syIn < INITIAL_LOOK_AHEAD)
{
//The highlighting seems slow in the first second of a speech, let's highlight 2 syllables
//ahead in the beginning of a speech
syIn = INITIAL_LOOK_AHEAD;
}
int enInc = startInd + arr.Length;
if (maxSyllableAhead > 0)
{
//Debug.WriteLine("Time pier syllable=" + timePerSyllable);
if (syIn > maxSyllableAhead)
{
syIn = (int)maxSyllableAhead;
}
}
string currentEndWord = "";
if (enInc > 0 && enInc <= allSpeechText.Length)
{
currentEndWord = allSpeechText[enInc - 1];
}
Boolean cont = true;
while (syIn > 0 && cont)
{
if (enInc < syllableArray.Length)
{
string guessWord = allSpeechText[enInc];
//if the current end word has pause punctuation, stop look ahead
if (EBookUtil.containPausePunctuation(currentEndWord))
{
Debug.WriteLine("currentEndWord \"" + currentEndWord + "\" contains pause, stop look ahead");
break;
}
else if (EBookUtil.containPausePunctuation(guessWord))
{
//reduce 4 syllables from enInc
syIn -= 3;
//no guess ahead if there is any possible pause in guess ahead word.
//Debug.WriteLine("guessWord \"" + guessWord + "\" contains pause, stop look ahead");
//cont = false;
}
if (syIn >= syllableArray[enInc])
{
increment++;
}
syIn -= syllableArray[enInc];
enInc++;
}
else
{
break;
}
}
Debug.WriteLine("guess " + increment + " word(s) ahead");
}
//ONLY DISPLAY INTERMIDIATE RESULT WITH CONFIDENCE SCORE GREATER THAN HYP_THRESHOLD
if (confidence > HYP_THRESHOLD)
{
if (startInd == lastSegPosition) //continue with last segment
{
startIndex = confirmedStartIndex;
hypothesisEndIndex = arr.Length + lastSegPosition - 1;
endIndex = hypothesisEndIndex + increment;
}
else if (arr.Length > 1) //jumping to onther sentence when at least two word in the speech
{
int gap = startInd - confirmedEndIndex;
if (gap > 0 && gap <= SPEECH_JUMP_GAP_THRESHOLD)
{
//if reader skip maximum of SPEECH_JUMP_GAP_THRESHOLD, consider
//the continue highlight the sentence
}
else
{
startIndex = startInd;
}
hypothesisEndIndex = arr.Length + startInd - 1;
endIndex = hypothesisEndIndex + increment;
}
}
}
else
{
//duration != -1 only when !isH, so we need to estimate the time
//per syllable for the next scentence using the previous speech
int syCount = 0;
for (int i = startInd; i < arr.Length + startInd; i++)
{
syCount += syllableArray[i];
}
timePerSyllable = duration / syCount;
Trace.WriteLine("timePerSyllable: " + timePerSyllable);
maxSyllableAhead = (int)(maxLag / timePerSyllable); //
numOfHypothsis = 0;
if (startInd == lastSegPosition)
{
//if number of confirmed word in the complete speech is one word less than imcomplete speech,
//we can treat the last imcomplete speech as complete
int completeEndIndex = arr.Length + lastSegPosition - 1;
if (endIndex - completeEndIndex != 1)
{
endIndex = completeEndIndex;
}
lastSegPosition = endIndex + 1;
}
else
{
int gap = startInd - lastSegPosition;
if (gap > 0 && gap <= SPEECH_JUMP_GAP_THRESHOLD)
{
//if reader skip maximum of SPEECH_JUMP_GAP_THRESHOLD, consider
//the continue highlight the sentence
}
else
{
startIndex = startInd;
}
endIndex = arr.Length + startInd - 1;
lastSegPosition = endIndex + 1;
}
confirmedStartIndex = startIndex;
confirmedEndIndex = endIndex;
confirmedLastSegPosition = lastSegPosition;
if (mode != Mode.REPLAY)
{
//EBookSRDevice rec = EBookSRDevice.GetInstance();
//reach the end of the page
if (confirmedEndIndex == allSpeechText.Length - 1)
{
//stop SR when transitioning to the next page
//rec.enableSR(false);
//ret = 1;
//AbstractEBookEvent.raise(new FinishPageEvent());
}
else
{
string cfgPath = storyDirectory + "\\" + GRAMMAR_TMP_DIR + "\\" +
currentPage.GetPageNumber() + "_" + (endIndex + 1) + ".cfg";
if (!defaultStartIndexes.Contains(endIndex + 1))
{
//rec.enableSR(false);
//rec.UnloadOnGoingGrammar();
Grammar g = new Grammar(cfgPath);
g.Weight = EBookConstant.NEXT_WORD_WEIGHT;
g.Priority = EBookConstant.NEXT_WORD_PRIORITY;
//rec.LoadOnGoingGrammar(g);
ActivityExecutor.add(new InternalReloadOnGoingGrammarActivity(g));
Debug.WriteLine("Load onGoing grammar " + cfgPath);
//rec.GenerateAndLoadOnGoingGrammars(allSpeechText,endIndex+1);
//rec.enableSR(true);
}
else
{
//the next sentence has higher priority
//rec.ReloadAndChangeGrammarPriority(endIndex + 1);
ActivityExecutor.add(new InternalChangeGrammarPriorityActivity(endIndex + 1));
}
}
}
if (mode != Mode.RECORD && confirmedEndIndex == allSpeechText.Length - 1)
{
//the complete recognized result reaches the end of the page
ret = 1;
}
//Detect speech and complete
//AbstractEBookEvent.raise(new CompleteSpeechEvent());
}
Debug.WriteLine("startIndex=" + startIndex);
Debug.WriteLine("endIndex=" + endIndex);
Debug.WriteLine("lastSegPosition=" + lastSegPosition);
Debug.WriteLine("confirmedStartIndex=" + confirmedStartIndex);
Debug.WriteLine("confirmedEndIndex=" + confirmedEndIndex);
Debug.WriteLine("confirmedLastSegPosition=" + confirmedLastSegPosition);
return(ret);
}
}
/*
* If Mode == REAL TIME, enable jump to different sentence. otherwise, disable jump to different sentence
*/
public void process(Page page, Mode mode)
{
this.mode = mode;
currentPage = page;
//reset parameters when process a new page
resetParameters();
//retrieve data from cache
if (enableCache && pageTextCached.ContainsKey(page.GetPageNumber()))
{
pageTextCached.TryGetValue(page.GetPageNumber(), out allSpeechText);
pageTextHTMLCached.TryGetValue(page.GetPageNumber(), out allSpeechTextHTML);
annotationArrayCached.TryGetValue(page.GetPageNumber(), out annotationArray);
syllableArrayCached.TryGetValue(page.GetPageNumber(), out syllableArray);
}
else
{
List <string[]> pageText = page.GetListOfTextArray();
List <string> annotation = page.GetListOfAnnotations();
List <string> annArray = new List <string>();
//cover list to array
string whole = "";
string wholeHTML = "";
//obtain the text and the text in HTML format for the current page
for (int i = 0; i < pageText.Count; i++)
{
if (pageText.ElementAt(i) == null)
{
wholeHTML = wholeHTML.TrimEnd() + "<br> ";
}
else
{
foreach (string str in pageText.ElementAt(i))
{
whole += str + " ";
if (str.Trim().Length == 0)
{
wholeHTML = wholeHTML.TrimEnd() + " ";
}
else
{
wholeHTML += str + " ";
}
annArray.Add(annotation.ElementAt(i));
}
//wholeHTML = wholeHTML.TrimEnd() + "<br> ";
}
}
whole = whole.Replace("\"", "");
allSpeechText = whole.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
allSpeechTextHTML = wholeHTML.Split(default(string[]), StringSplitOptions.RemoveEmptyEntries);
annotationArray = annArray.ToArray();
syllableArray = EBookUtil.CountSyllables(allSpeechText);
//save the data to hash map, we can simply retrieve the page data when
//the page get revisit again
if (enableCache)
{
pageTextCached.Add(page.GetPageNumber(), allSpeechText);
pageTextHTMLCached.Add(page.GetPageNumber(), allSpeechTextHTML);
annotationArrayCached.Add(page.GetPageNumber(), annotationArray);
syllableArrayCached.Add(page.GetPageNumber(), syllableArray);
}
}
if (mode != Mode.REPLAY)
{
defaultStartIndexes.Clear();
List <SrgsDocument> srgsDocs = EBookUtil.GenerateGrammars(allSpeechText, annotationArray);
Debug.WriteLine("generated " + srgsDocs.Count + " grammar files");
//load grammars
List <Grammar> gs = new List <Grammar>();
//loop from srgsDocs.Count to 0 will give the early sentence the priority.
for (int i = srgsDocs.Count; --i >= 0;)
{
string cfgPath = storyDirectory + "\\" + GRAMMAR_TMP_DIR + "\\" + page.GetPageNumber() + "_" + i + ".cfg";
Directory.CreateDirectory(storyDirectory + "\\" + GRAMMAR_TMP_DIR);
CompileGrammar(srgsDocs.ElementAt(i), cfgPath);
if (mode == Mode.REALTIME)
{
if (i == 0 || (i > 0 && (allSpeechText[i - 1].Contains("?") || allSpeechText[i - 1].Contains(".") ||
allSpeechText[i - 1].Contains("!"))))
{
defaultStartIndexes.Add(i);
Debug.WriteLine("loading grammar:" + cfgPath);
Grammar storyG = new Grammar(cfgPath);
storyG.Weight = EBookConstant.DEFAULT_WEIGHT;
storyG.Priority = EBookConstant.DEFAULT_PRIORITY;
gs.Add(storyG);
}
}
else
{
if (i == 0)
{
defaultStartIndexes.Add(i);
Debug.WriteLine("loading grammar:" + cfgPath);
Grammar storyG = new Grammar(cfgPath);
storyG.Weight = EBookConstant.DEFAULT_WEIGHT;
storyG.Priority = EBookConstant.DEFAULT_PRIORITY;
ActivityExecutor.add(new InternalReloadOnGoingGrammarActivity(storyG));
}
}
}
if (gs.Count > 0)
{
ActivityExecutor.add(new InternalReloadStoryGrammarActivity(gs));
}
}
string pageTextStr = constructPageText();
ActivityExecutor.add(new InternalUpdatePageTextActivity(pageTextStr, page.pageNumber));
}