private void MainWindow_Shown(object sender, EventArgs e) { // Create the \psi pipeline this.pipeline = Pipeline.Create(); // Create the webcam component var webcam = new MediaCapture(this.pipeline, 640, 480, "/dev/video0", PixelFormatId.YUYV); // Create the audio capture component var audio = new AudioCapture(this.pipeline, new AudioCaptureConfiguration { DeviceName = "plughw:0,0", Format = WaveFormat.Create16kHz1Channel16BitPcm() }); // Create an acoustic features extractor component and pipe the audio to it var acousticFeatures = new AcousticFeaturesExtractor(this.pipeline); audio.PipeTo(acousticFeatures); // Fuse the webcam images with the audio log energy level var webcamWithAudioEnergy = webcam.Join(acousticFeatures.LogEnergy, RelativeTimeInterval.Past()); // Overlay the audio energy on the webcam image and display it in the window. // The "Do" operator is executed on each fused webcam and audio energy sample. webcamWithAudioEnergy.Do( frame => { // Update the window with the latest frame this.DrawFrame(frame); }, DeliveryPolicy.LatestMessage); // Start the pipeline running this.pipeline.RunAsync(); }
/// <summary> /// Builds and runs a speech recognition pipeline using the Azure speech recognizer. Requires a valid Cognitive Services /// subscription key. See https://docs.microsoft.com/en-us/azure/cognitive-services/cognitive-services-apis-create-account. /// </summary> /// <remarks> /// If you are getting a <see cref="System.InvalidOperationException"/> with the message 'AzureSpeechRecognizer returned /// OnConversationError with error code: LoginFailed. Original error text: Transport error', this most likely is due to /// an invalid subscription key. Please check your Azure portal at https://portal.azure.com and ensure that you have /// added a subscription to the Azure Speech API on your account. /// </remarks> public static void RunAzureSpeech() { // Create the pipeline object. using (Pipeline pipeline = Pipeline.Create()) { // Create the AudioSource component to capture audio from the default device in 16 kHz 1-channel // PCM format as required by both the voice activity detector and speech recognition components. IProducer <AudioBuffer> audioInput = new AudioCapture(pipeline, new AudioCaptureConfiguration() { DeviceName = "plughw:0,0", Format = WaveFormat.Create16kHz1Channel16BitPcm() }); // Perform voice activity detection using the voice activity detector component var vad = new SimpleVoiceActivityDetector(pipeline); audioInput.PipeTo(vad); // Create Azure speech recognizer component var recognizer = new AzureSpeechRecognizer(pipeline, new AzureSpeechRecognizerConfiguration() { SubscriptionKey = Program.azureSubscriptionKey, Region = Program.azureRegion }); // The input audio to the Azure speech recognizer needs to be annotated with a voice activity flag. // This can be constructed by using the Psi Join() operator to combine the audio and VAD streams. var annotatedAudio = audioInput.Join(vad); // Subscribe the recognizer to the annotated audio annotatedAudio.PipeTo(recognizer); // Partial and final speech recognition results are posted on the same stream. Here // we use Psi's Where() operator to filter out only the final recognition results. var finalResults = recognizer.Out.Where(result => result.IsFinal); // Print the recognized text of the final recognition result to the console. finalResults.Do(result => Console.WriteLine(result.Text)); // Register an event handler to catch pipeline errors pipeline.PipelineExceptionNotHandled += Pipeline_PipelineException; // Register an event handler to be notified when the pipeline completes pipeline.PipelineCompleted += Pipeline_PipelineCompleted; // Run the pipeline pipeline.RunAsync(); // Azure speech transcribes speech to text Console.WriteLine("Say anything"); Console.WriteLine("Press any key to exit..."); Console.ReadKey(true); } }
static void Main(string[] args) { using (Pipeline pipeline = Pipeline.Create()) { WaveFormat waveFormat = WaveFormat.Create16kHz1Channel16BitPcm(); IProducer <AudioBuffer> audioInput = new AudioCapture(pipeline, new AudioCaptureConfiguration() { OutputFormat = waveFormat }); DataFaucet <AudioBuffer> df = new DataFaucet <AudioBuffer>(pipeline); audioInput.PipeTo(df); AggregateDump dump = new AggregateDump(pipeline); df.PipeTo(dump); GoogleASR gsr = new GoogleASR(pipeline, "en"); //gsr for google speech recognition dump.PipeTo(gsr); GoogleTranslate gt = new GoogleTranslate(pipeline, "en", "de"); //gt for google translate gsr.PipeTo(gt); GoogleSpeak gs = new GoogleSpeak(pipeline, waveFormat, "de-DE"); //gs for google speak gt.PipeTo(gs); AudioOutput aOut = new AudioOutput(pipeline); //aOut for audio out gs.PipeTo(aOut); ActiveMQ rasa = new ActiveMQ(pipeline, "rasa.PSI", "rasa.PYTHON"); gsr.PipeTo(rasa); GUI gui = new GUI(df, dump, gsr, gt); Thread thread = new Thread(() => { gui.ShowDialog(); }); thread.Start(); pipeline.RunAsync(); Console.ReadKey(true); } }
/// <summary> /// Builds and runs a speech recognition pipeline using the Azure speech service. Requires a valid Cognitive Services /// subscription key. See https://docs.microsoft.com/en-us/azure/cognitive-services/speech-service/get-started. /// </summary> public static void RunAzureSpeech() { // Create the pipeline object. using (Pipeline pipeline = Pipeline.Create()) { // Create the AudioSource component to capture audio from the default device in 16 kHz 1-channel // PCM format as required by the speech recognition component. var audio = new AudioCapture(pipeline, new AudioCaptureConfiguration { DeviceName = deviceName, Format = WaveFormat.Create16kHz1Channel16BitPcm() }); // Create the speech recognizer component var recognizer = new ContinuousSpeechRecognizer(pipeline, azureSubscriptionKey, azureRegion); // Subscribe the recognizer to the annotated audio audio.PipeTo(recognizer); // Print the recognized text of the final recognition result to the console. recognizer.Out.Do((result, e) => Console.WriteLine($"{e.OriginatingTime.TimeOfDay}: {result}")); // Register an event handler to catch pipeline errors pipeline.PipelineExceptionNotHandled += Pipeline_PipelineException; // Register an event handler to be notified when the pipeline completes pipeline.PipelineCompleted += Pipeline_PipelineCompleted; // Run the pipeline pipeline.RunAsync(); // Azure speech transcribes speech to text Console.WriteLine("Say anything"); Console.WriteLine("Press any key to exit..."); Console.ReadKey(true); } }