private void MainWindow_Shown(object sender, EventArgs e)
{
// Create the \psi pipeline
this.pipeline = Pipeline.Create();
// Create the webcam component
var webcam = new MediaCapture(this.pipeline, 640, 480, "/dev/video0", PixelFormatId.YUYV);
// Create the audio capture component
var audio = new AudioCapture(this.pipeline, new AudioCaptureConfiguration {
DeviceName = "plughw:0,0", Format = WaveFormat.Create16kHz1Channel16BitPcm()
});
// Create an acoustic features extractor component and pipe the audio to it
var acousticFeatures = new AcousticFeaturesExtractor(this.pipeline);
audio.PipeTo(acousticFeatures);
// Fuse the webcam images with the audio log energy level
var webcamWithAudioEnergy = webcam.Join(acousticFeatures.LogEnergy, RelativeTimeInterval.Past());
// Overlay the audio energy on the webcam image and display it in the window.
// The "Do" operator is executed on each fused webcam and audio energy sample.
webcamWithAudioEnergy.Do(
frame =>
{
// Update the window with the latest frame
this.DrawFrame(frame);
},
DeliveryPolicy.LatestMessage);
// Start the pipeline running
this.pipeline.RunAsync();
}
/// <summary>
/// Builds and runs a speech recognition pipeline using the Azure speech recognizer. Requires a valid Cognitive Services
/// subscription key. See https://docs.microsoft.com/en-us/azure/cognitive-services/cognitive-services-apis-create-account.
/// </summary>
/// <remarks>
/// If you are getting a <see cref="System.InvalidOperationException"/> with the message 'AzureSpeechRecognizer returned
/// OnConversationError with error code: LoginFailed. Original error text: Transport error', this most likely is due to
/// an invalid subscription key. Please check your Azure portal at https://portal.azure.com and ensure that you have
/// added a subscription to the Azure Speech API on your account.
/// </remarks>
public static void RunAzureSpeech()
{
// Create the pipeline object.
using (Pipeline pipeline = Pipeline.Create())
{
// Create the AudioSource component to capture audio from the default device in 16 kHz 1-channel
// PCM format as required by both the voice activity detector and speech recognition components.
IProducer <AudioBuffer> audioInput = new AudioCapture(pipeline, new AudioCaptureConfiguration()
{
DeviceName = "plughw:0,0", Format = WaveFormat.Create16kHz1Channel16BitPcm()
});
// Perform voice activity detection using the voice activity detector component
var vad = new SimpleVoiceActivityDetector(pipeline);
audioInput.PipeTo(vad);
// Create Azure speech recognizer component
var recognizer = new AzureSpeechRecognizer(pipeline, new AzureSpeechRecognizerConfiguration()
{
SubscriptionKey = Program.azureSubscriptionKey, Region = Program.azureRegion
});
// The input audio to the Azure speech recognizer needs to be annotated with a voice activity flag.
// This can be constructed by using the Psi Join() operator to combine the audio and VAD streams.
var annotatedAudio = audioInput.Join(vad);
// Subscribe the recognizer to the annotated audio
annotatedAudio.PipeTo(recognizer);
// Partial and final speech recognition results are posted on the same stream. Here
// we use Psi's Where() operator to filter out only the final recognition results.
var finalResults = recognizer.Out.Where(result => result.IsFinal);
// Print the recognized text of the final recognition result to the console.
finalResults.Do(result => Console.WriteLine(result.Text));
// Register an event handler to catch pipeline errors
pipeline.PipelineExceptionNotHandled += Pipeline_PipelineException;
// Register an event handler to be notified when the pipeline completes
pipeline.PipelineCompleted += Pipeline_PipelineCompleted;
// Run the pipeline
pipeline.RunAsync();
// Azure speech transcribes speech to text
Console.WriteLine("Say anything");
Console.WriteLine("Press any key to exit...");
Console.ReadKey(true);
}
}
static void Main(string[] args)
{
using (Pipeline pipeline = Pipeline.Create())
{
WaveFormat waveFormat = WaveFormat.Create16kHz1Channel16BitPcm();
IProducer <AudioBuffer> audioInput = new AudioCapture(pipeline, new AudioCaptureConfiguration()
{
OutputFormat = waveFormat
});
DataFaucet <AudioBuffer> df = new DataFaucet <AudioBuffer>(pipeline);
audioInput.PipeTo(df);
AggregateDump dump = new AggregateDump(pipeline);
df.PipeTo(dump);
GoogleASR gsr = new GoogleASR(pipeline, "en"); //gsr for google speech recognition
dump.PipeTo(gsr);
GoogleTranslate gt = new GoogleTranslate(pipeline, "en", "de"); //gt for google translate
gsr.PipeTo(gt);
GoogleSpeak gs = new GoogleSpeak(pipeline, waveFormat, "de-DE"); //gs for google speak
gt.PipeTo(gs);
AudioOutput aOut = new AudioOutput(pipeline); //aOut for audio out
gs.PipeTo(aOut);
ActiveMQ rasa = new ActiveMQ(pipeline, "rasa.PSI", "rasa.PYTHON");
gsr.PipeTo(rasa);
GUI gui = new GUI(df, dump, gsr, gt);
Thread thread = new Thread(() =>
{
gui.ShowDialog();
});
thread.Start();
pipeline.RunAsync();
Console.ReadKey(true);
}
}
/// <summary>
/// Builds and runs a speech recognition pipeline using the Azure speech service. Requires a valid Cognitive Services
/// subscription key. See https://docs.microsoft.com/en-us/azure/cognitive-services/speech-service/get-started.
/// </summary>
public static void RunAzureSpeech()
{
// Create the pipeline object.
using (Pipeline pipeline = Pipeline.Create())
{
// Create the AudioSource component to capture audio from the default device in 16 kHz 1-channel
// PCM format as required by the speech recognition component.
var audio = new AudioCapture(pipeline, new AudioCaptureConfiguration {
DeviceName = deviceName, Format = WaveFormat.Create16kHz1Channel16BitPcm()
});
// Create the speech recognizer component
var recognizer = new ContinuousSpeechRecognizer(pipeline, azureSubscriptionKey, azureRegion);
// Subscribe the recognizer to the annotated audio
audio.PipeTo(recognizer);
// Print the recognized text of the final recognition result to the console.
recognizer.Out.Do((result, e) => Console.WriteLine($"{e.OriginatingTime.TimeOfDay}: {result}"));
// Register an event handler to catch pipeline errors
pipeline.PipelineExceptionNotHandled += Pipeline_PipelineException;
// Register an event handler to be notified when the pipeline completes
pipeline.PipelineCompleted += Pipeline_PipelineCompleted;
// Run the pipeline
pipeline.RunAsync();
// Azure speech transcribes speech to text
Console.WriteLine("Say anything");
Console.WriteLine("Press any key to exit...");
Console.ReadKey(true);
}
}