public void SynthesizeText(string text,
List <AudioMediaBuffer> audioMediaBuffers,
List <VideoMediaBuffer> videoMediaBuffers)
{
// stream for the output audio
var audioStream = new MemoryStream();
// set the synthesizer output to the stream, make sure the output format is matching the audio socket settings
_synth.SetOutputToAudioStream(audioStream,
new SpeechAudioFormatInfo(samplesPerSecond: 16000, bitsPerSample: AudioBitsPerSample.Sixteen,
channel: AudioChannel.Mono));
// observe the synthesizer to generate the visemes timeline
VisemesTimeline timeline = new VisemesTimeline();
_synth.VisemeReached += (sender, visemeReachedEventArgs) =>
{
timeline.Add(visemeReachedEventArgs.Viseme, visemeReachedEventArgs.Duration.Milliseconds);
};
// synthesize the text -> audio and visemes are generated
_synth.Speak(text);
// geneate the buffers and synchronize them with the current time
long referenceTimeTick = DateTime.Now.Ticks;
CreateAudioBuffers(audioStream, audioMediaBuffers, referenceTimeTick);
CreateVideoBuffers(timeline, videoMediaBuffers, referenceTimeTick);
}
/// <summary>
/// Create video buffers from a viseme timeline and populate the given list starting at the reference time tick.
/// </summary>
/// <param name="visemesTimeline">The viseme timeline</param>
/// <param name="videoBuffers">The list of video buffers to be populated</param>
/// <param name="referenceTimeTick">The reference starting time tick</param>
private void CreateVideoBuffers(VisemesTimeline visemesTimeline, List <VideoMediaBuffer> videoBuffers,
long referenceTimeTick)
{
// compute the frame buffer size in bytes for the current video format
var frameSize = (int)(_videoFormat.Width * _videoFormat.Height *
Helper.GetBitsPerPixel(_videoFormat.VideoColorFormat) / 8);
// compute the frame duration for the current framerate
var frameDurationInMs = (int)(1000.0 / (double)_videoFormat.FrameRate);
var durationInMs = 0;
// create video frames for the whole viseme timeline lenght
while (durationInMs < visemesTimeline.Length)
{
// get the current viseme
byte[] visemeBitmap = _visemeBitmaps[visemesTimeline.Get(durationInMs)];
// create the buffer
IntPtr unmanagedBuffer = Marshal.AllocHGlobal(frameSize);
Marshal.Copy(visemeBitmap, 0, unmanagedBuffer, frameSize);
// increase the current duration by one frame
durationInMs += frameDurationInMs;
// create the video buffer and add it to the list
var videoSendBuffer = new VideoSendBuffer(unmanagedBuffer, (uint)frameSize,
_videoFormat, referenceTimeTick + durationInMs * 10000);
videoBuffers.Add(videoSendBuffer);
}
Log.Info(
new CallerInfo(),
LogContext.Media,
"created {0} VideoMediaBuffers frames", videoBuffers.Count);
}
