public void SynthesizeText(string text, List <AudioMediaBuffer> audioMediaBuffers, List <VideoMediaBuffer> videoMediaBuffers) { // stream for the output audio var audioStream = new MemoryStream(); // set the synthesizer output to the stream, make sure the output format is matching the audio socket settings _synth.SetOutputToAudioStream(audioStream, new SpeechAudioFormatInfo(samplesPerSecond: 16000, bitsPerSample: AudioBitsPerSample.Sixteen, channel: AudioChannel.Mono)); // observe the synthesizer to generate the visemes timeline VisemesTimeline timeline = new VisemesTimeline(); _synth.VisemeReached += (sender, visemeReachedEventArgs) => { timeline.Add(visemeReachedEventArgs.Viseme, visemeReachedEventArgs.Duration.Milliseconds); }; // synthesize the text -> audio and visemes are generated _synth.Speak(text); // geneate the buffers and synchronize them with the current time long referenceTimeTick = DateTime.Now.Ticks; CreateAudioBuffers(audioStream, audioMediaBuffers, referenceTimeTick); CreateVideoBuffers(timeline, videoMediaBuffers, referenceTimeTick); }
/// <summary> /// Create video buffers from a viseme timeline and populate the given list starting at the reference time tick. /// </summary> /// <param name="visemesTimeline">The viseme timeline</param> /// <param name="videoBuffers">The list of video buffers to be populated</param> /// <param name="referenceTimeTick">The reference starting time tick</param> private void CreateVideoBuffers(VisemesTimeline visemesTimeline, List <VideoMediaBuffer> videoBuffers, long referenceTimeTick) { // compute the frame buffer size in bytes for the current video format var frameSize = (int)(_videoFormat.Width * _videoFormat.Height * Helper.GetBitsPerPixel(_videoFormat.VideoColorFormat) / 8); // compute the frame duration for the current framerate var frameDurationInMs = (int)(1000.0 / (double)_videoFormat.FrameRate); var durationInMs = 0; // create video frames for the whole viseme timeline lenght while (durationInMs < visemesTimeline.Length) { // get the current viseme byte[] visemeBitmap = _visemeBitmaps[visemesTimeline.Get(durationInMs)]; // create the buffer IntPtr unmanagedBuffer = Marshal.AllocHGlobal(frameSize); Marshal.Copy(visemeBitmap, 0, unmanagedBuffer, frameSize); // increase the current duration by one frame durationInMs += frameDurationInMs; // create the video buffer and add it to the list var videoSendBuffer = new VideoSendBuffer(unmanagedBuffer, (uint)frameSize, _videoFormat, referenceTimeTick + durationInMs * 10000); videoBuffers.Add(videoSendBuffer); } Log.Info( new CallerInfo(), LogContext.Media, "created {0} VideoMediaBuffers frames", videoBuffers.Count); }