/// <summary>
/// Starts animating the next viseme in the current set of visemes.
///
/// Empties the set of visemes and resets the currently playing viseme when
/// all visemes in the set have been animated.
/// </summary>
private void playNextViseme()
{
// if all visemes in the set have been animated
if (currentVisemeInList >= visemeAmount)
{
stopSpeechAnimation(charIndex);
return;
}
Viseme current = visemeList[currentVisemeInList];
currentVisemeName = current.getVisemeCode().getName();
currentVisemeLength = (float)current.getDuration();
// if punctuation was encountered before this silence, pause untill next
// onboundary event. Only applied if onboundary events are supported.
if (shouldPause && currentVisemeName == "Silence")
{
isSpeaking = false;
shouldPause = false;
animator.CrossFade("Silence", 0, visemeLayer);
currentVisemeInList++;
return;
}
// play the found viseme animation
if (animator != null)
{
animator.CrossFade(currentVisemeName, 0, visemeLayer);
}
// increment currently playing viseme
currentVisemeInList++;
}
private void AssignCurrentVisemeData()
{
currentViseme = (Viseme)visemes[currentVisemeIndex];
currentBlendShapeIndex = BlendShapeInfo.GetBlendShapeIndex(currentViseme);
currentVisemeStartTime = currentViseme.startTime;
currentVisemeEndTime = currentViseme.endTime;
}
private void OnVisemeHit(Viseme v)
{
if (_activeVoicePlayer == null)
{
return;
}
SetVisemeDisplay(v);
}
public static int GetBlendShapeIndex(Viseme viseme)
{
if (viseme is Viseme_OO)
{
return(0);
}
else if (viseme is Viseme_FV)
{
return(1);
}
else if (viseme is Viseme_EE)
{
return(2);
}
else if (viseme is Viseme_BM)
{
return(3);
}
else if (viseme is Viseme_R)
{
return(4);
}
else if (viseme is Viseme_IH)
{
return(5);
}
else if (viseme is Viseme_CH)
{
return(6);
}
else if (viseme is Viseme_AA)
{
return(7);
}
else if (viseme is Viseme_UU)
{
return(8);
}
else
{
return(0);
}
}
private void SetVisemeDisplay(Viseme v)
{
Bitmap b;
switch (v)
{
case Viseme.Silence:
b = Resources._0_silence;
break;
case Viseme.AxAhUh:
b = Resources._1_11_AxAhUh;
break;
case Viseme.Aa:
b = Resources._2_Aa;
break;
case Viseme.Ao:
b = Resources._3_Ao;
break;
case Viseme.EyEhAe:
b = Resources._4_EyEhAe;
break;
case Viseme.Er:
b = Resources._5_Er;
break;
case Viseme.YIyIhIx:
b = Resources._6_YIyIhIx;
break;
case Viseme.WUwU:
b = Resources._7_WUwU;
break;
case Viseme.Ow:
b = Resources._8_10_Ow;
break;
case Viseme.Aw:
b = Resources._9_Aw;
break;
case Viseme.Oy:
b = Resources._8_10_Ow;
break;
case Viseme.Ay:
b = Resources._9_Aw;
break;
case Viseme.H:
b = Resources._12_H;
break;
case Viseme.R:
b = Resources._13_R;
break;
case Viseme.L:
b = Resources._14_L;
break;
case Viseme.SZTs:
b = Resources._15_SZTs;
break;
case Viseme.ShChJhZh:
b = Resources._16_ShChJhZh;
break;
case Viseme.ThDh:
b = Resources._17_ThDh;
break;
case Viseme.FV:
b = Resources._18_FV;
break;
case Viseme.DTDxN:
b = Resources._19_DTDxN;
break;
case Viseme.KGNg:
b = Resources._20_KGNg;
break;
case Viseme.PBM:
b = Resources._21_PBM;
break;
default:
throw new ArgumentOutOfRangeException(nameof(v), v, null);
}
_visemeDisplay.Image = b;
}
protected override BakedTTS BuildTTS(IVoice voice, string text, double rate, int pitch)
{
var tts = RequestTTS((L2FVoice)voice, text);
if (tts == null)
{
return(null);
}
//Create visemes
List <VisemeSpan> visemes = new List <VisemeSpan>();
//Convert phones to visemes and adjust time to rate
var mapper = LanguageResources.PT_PhonesToVisemes;
Viseme lastViseme = Viseme.Silence;
double totalVisemeTime = 0;
foreach (var r in tts)
{
var phones = ((JsonArray)r["phones"]).Select(t => ((JsonString)t).String);
var time = ((JsonArray)r["times"]).Select(t => Convert.ToDouble(((JsonNumber)t).Value));
var toProcess = phones.Zip(time, (s, d) => new { ph = s, entryTime = d }).ToArray();
totalVisemeTime += toProcess[0].entryTime; ///rate;
lastViseme = Viseme.Silence;
for (int i = 1; i < toProcess.Length - 1; i++)
{
var p = toProcess[i];
var nextTime = toProcess[i + 1].entryTime;
Viseme currentViseme;
if (!mapper.TryGetValue(p.ph, out currentViseme))
{
currentViseme = Viseme.Silence;
}
if (lastViseme != currentViseme)
{
visemes.Add(new VisemeSpan()
{
duration = totalVisemeTime / rate, viseme = lastViseme
});
lastViseme = currentViseme;
totalVisemeTime = 0;
}
var duration = nextTime - p.entryTime;
totalVisemeTime += duration;
}
}
if (totalVisemeTime > 0)
{
visemes.Add(new VisemeSpan()
{
duration = totalVisemeTime / rate, viseme = lastViseme
});
}
if (visemes.Count == 0)
{
return(null);
}
//Create audio
var audioUrls = tts.Select(r => ((JsonString)r["url"]).String).ToArray();
var v = BuildAudioStream(audioUrls, rate, pitch);
const int MAX_SAMPLES = 5120;
var bufferSize = MAX_SAMPLES * v.WaveFormat.Channels;
var buffer = new float[bufferSize];
using (var m = new MemoryStream())
{
using (var w = new WaveFileWriter(m, v.WaveFormat))
{
int readed;
while ((readed = v.Read(buffer, 0, bufferSize)) > 0)
{
w.WriteSamples(buffer, 0, readed);
}
w.Flush();
return(new BakedTTS()
{
visemes = visemes.ToArray(), waveStreamData = m.ToArray()
});
}
}
}
