/// <summary> /// Print (first n) messages from stream. /// </summary> /// <param name="stream">Stream name.</param> /// <param name="store">Store name.</param> /// <param name="path">Store path.</param> /// <param name="number">Number of messages to display.</param> /// <returns>Success flag.</returns> internal static int DisplayStreamMessages(string stream, string store, string path, int number) { Console.WriteLine($"Stream Messages (stream={stream}, store={store}, path={path}, number={number})"); using (var pipeline = Pipeline.Create()) { var count = 0; var data = PsiStore.Open(pipeline, store, Path.GetFullPath(path)); data.OpenDynamicStream(stream).Do((m, e) => { if (count++ < number) { PrintMessage(m, e); } }); pipeline.RunAsync(); while (count < number) { Thread.Sleep(100); } } return(0); }
public async Task SessionCreateDerivedPartitionCancellation() { var dataset = new Dataset(); var session = dataset.CreateSession(); // generate a test store var originalStoreName = "OriginalStore"; GenerateTestStore(originalStoreName, StorePath); // add a partition var partition0 = session.AddPsiStorePartition(originalStoreName, StorePath, "Partition_0"); Assert.AreEqual(1, session.Partitions.Count); Assert.AreEqual("Partition_0", session.Partitions[0].Name); int multiplier = 7; try { // create a cancellation token source that will automatically request cancellation after 500 ms var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); // create a derived partition which contains the values from the original stream multiplied by a multiplier await session.CreateDerivedPsiPartitionAsync( (pipeline, importer, exporter, parameter) => { var inputStream = importer.OpenStream <int>("Root"); var derivedStream = inputStream.Sample(TimeSpan.FromMinutes(1), RelativeTimeInterval.Infinite).Select(x => x *parameter).Write("DerivedStream", exporter); // add a dummy source and propose a long time interval so that the operation will block (and eventually be canceled) var generator = Generators.Repeat(pipeline, 0, int.MaxValue, TimeSpan.FromMilliseconds(1000)); var replayTimeInterval = TimeInterval.LeftBounded(importer.MessageOriginatingTimeInterval.Left); pipeline.ProposeReplayTime(replayTimeInterval); }, multiplier, "Partition_1", false, "DerivedStore", StorePath, replayDescriptor : null, deliveryPolicy : null, enableDiagnostics : false, progress : null, cancellationToken : cts.Token); } catch (OperationCanceledException) { // should NOT have created a new partition (but original partition should be intact) Assert.AreEqual(1, session.Partitions.Count); Assert.IsTrue(PsiStore.Exists("OriginalStore", StorePath)); Assert.IsFalse(PsiStore.Exists("DerivedStore", StorePath)); // verify original partition metadata var originalPartition = session.Partitions[0] as Partition <PsiStoreStreamReader>; Assert.AreEqual("Partition_0", originalPartition.Name); Assert.AreEqual("OriginalStore", originalPartition.StoreName); Assert.AreEqual($"{StorePath}{Path.DirectorySeparatorChar}OriginalStore.0000", originalPartition.StorePath); throw; } }
public async Task SessionCreateDerivedPartition() { var dataset = new Dataset(); var session = dataset.CreateSession(); // generate a test store var originalStoreName = "OriginalStore"; GenerateTestStore(originalStoreName, StorePath); // add a partition var partition0 = session.AddPsiStorePartition(originalStoreName, StorePath, "Partition_0"); Assert.AreEqual(1, session.Partitions.Count); Assert.AreEqual("Partition_0", session.Partitions[0].Name); int multiplier = 7; // create a derived partition which contains the values from the original stream multiplied by a multiplier await session.CreateDerivedPsiPartitionAsync( (pipeline, importer, exporter, parameter) => { var inputStream = importer.OpenStream <int>("Root"); var derivedStream = inputStream.Select(x => x *parameter).Write("DerivedStream", exporter); }, multiplier, "Partition_1", false, "DerivedStore", StorePath); // should have created a new store partition Assert.AreEqual(2, session.Partitions.Count); Assert.IsTrue(PsiStore.Exists("OriginalStore", StorePath)); Assert.IsTrue(PsiStore.Exists("DerivedStore", StorePath)); // verify partition metadata var originalPartition = session.Partitions[0] as Partition <PsiStoreStreamReader>; Assert.AreEqual("Partition_0", originalPartition.Name); Assert.AreEqual("OriginalStore", originalPartition.StoreName); Assert.AreEqual($"{StorePath}{Path.DirectorySeparatorChar}OriginalStore.0000", originalPartition.StorePath); var derivedPartition = session.Partitions[1] as Partition <PsiStoreStreamReader>; Assert.AreEqual("Partition_1", derivedPartition.Name); Assert.AreEqual("DerivedStore", derivedPartition.StoreName); Assert.AreEqual(StorePath, derivedPartition.StorePath); // collections to capture stream values var originalValues = new List <int>(); var originalTimes = new List <DateTime>(); var derivedValues = new List <int>(); var derivedTimes = new List <DateTime>(); // read stream values from the partitions using (var pipeline = Pipeline.Create()) { var originalPartitionImporter = PsiStore.Open(pipeline, originalPartition.StoreName, originalPartition.StorePath); originalPartitionImporter.OpenStream <int>("Root").Do( (i, e) => { originalValues.Add(i); originalTimes.Add(e.OriginatingTime); }); var derivedPartitionImporter = PsiStore.Open(pipeline, derivedPartition.StoreName, derivedPartition.StorePath); derivedPartitionImporter.OpenStream <int>("DerivedStream").Do( (i, e) => { derivedValues.Add(i); derivedTimes.Add(e.OriginatingTime); }); pipeline.Run(); } // verify that we read the data Assert.AreEqual(10, originalValues.Count); Assert.AreEqual(10, originalTimes.Count); Assert.AreEqual(10, derivedValues.Count); Assert.AreEqual(10, derivedTimes.Count); // verify values from both streams are what we expect CollectionAssert.AreEqual(originalValues.Select(x => x * multiplier).ToList(), derivedValues); CollectionAssert.AreEqual(originalTimes, derivedTimes); }
/// <summary> /// Builds and runs a speech recognition pipeline using the Azure speech recognizer. Requires a valid Cognitive Services /// subscription key. See https://docs.microsoft.com/en-us/azure/cognitive-services/cognitive-services-apis-create-account. /// </summary> /// <remarks> /// If you are getting a <see cref="System.InvalidOperationException"/> with the message 'AzureSpeechRecognizer returned /// OnConversationError with error code: LoginFailed. Original error text: Transport error', this most likely is due to /// an invalid subscription key. Please check your Azure portal at https://portal.azure.com and ensure that you have /// added a subscription to the Azure Speech API on your account. /// </remarks> /// <param name="outputLogPath">The path under which to write log data.</param> /// <param name="inputLogPath">The path from which to read audio input data.</param> public static void RunAzureSpeech(string outputLogPath = null, string inputLogPath = null) { // Create the pipeline object. using (Pipeline pipeline = Pipeline.Create()) { // Use either live audio from the microphone or audio from a previously saved log IProducer <AudioBuffer> audioInput = null; if (inputLogPath != null) { // Open the MicrophoneAudio stream from the last saved log var store = PsiStore.Open(pipeline, Program.AppName, inputLogPath); audioInput = store.OpenStream <AudioBuffer>($"{Program.AppName}.MicrophoneAudio"); } else { // Create the AudioCapture component to capture audio from the default device in 16 kHz 1-channel // PCM format as required by both the voice activity detector and speech recognition components. audioInput = new AudioCapture(pipeline, WaveFormat.Create16kHz1Channel16BitPcm()); } // Perform voice activity detection using the voice activity detector component var vad = new SystemVoiceActivityDetector(pipeline); audioInput.PipeTo(vad); // Create Azure speech recognizer component var recognizer = new AzureSpeechRecognizer(pipeline, new AzureSpeechRecognizerConfiguration() { SubscriptionKey = Program.azureSubscriptionKey, Region = Program.azureRegion }); // The input audio to the Azure speech recognizer needs to be annotated with a voice activity flag. // This can be constructed by using the Psi Join() operator to combine the audio and VAD streams. var annotatedAudio = audioInput.Join(vad); // Subscribe the recognizer to the annotated audio annotatedAudio.PipeTo(recognizer); // Partial and final speech recognition results are posted on the same stream. Here // we use Psi's Where() operator to filter out only the final recognition results. var finalResults = recognizer.Out.Where(result => result.IsFinal); // Print the recognized text of the final recognition result to the console. finalResults.Do(result => Console.WriteLine(result.Text)); // Create a data store to log the data to if necessary. A data store is necessary // only if output logging is enabled. var dataStore = CreateDataStore(pipeline, outputLogPath); // For disk logging only if (dataStore != null) { // Log the microphone audio and recognition results audioInput.Write($"{Program.AppName}.MicrophoneAudio", dataStore); finalResults.Write($"{Program.AppName}.FinalRecognitionResults", dataStore); vad.Write($"{Program.AppName}.VoiceActivity", dataStore); } // Register an event handler to catch pipeline errors pipeline.PipelineExceptionNotHandled += Pipeline_PipelineException; // Register an event handler to be notified when the pipeline completes pipeline.PipelineCompleted += Pipeline_PipelineCompleted; // Run the pipeline pipeline.RunAsync(); // Azure speech transcribes speech to text Console.WriteLine("Say anything"); Console.WriteLine("Press any key to exit..."); Console.ReadKey(true); } }
/// <summary> /// Builds and runs a speech recognition pipeline using the .NET System.Speech recognizer and a set of fixed grammars. /// </summary> /// <param name="outputLogPath">The path under which to write log data.</param> /// <param name="inputLogPath">The path from which to read audio input data.</param> public static void RunSystemSpeech(string outputLogPath = null, string inputLogPath = null) { // Create the pipeline object. using (Pipeline pipeline = Pipeline.Create()) { // Use either live audio from the microphone or audio from a previously saved log IProducer <AudioBuffer> audioInput = null; if (inputLogPath != null) { // Open the MicrophoneAudio stream from the last saved log var store = PsiStore.Open(pipeline, Program.AppName, inputLogPath); audioInput = store.OpenStream <AudioBuffer>($"{Program.AppName}.MicrophoneAudio"); } else { // Create the AudioCapture component to capture audio from the default device in 16 kHz 1-channel // PCM format as required by both the voice activity detector and speech recognition components. audioInput = new AudioCapture(pipeline, WaveFormat.Create16kHz1Channel16BitPcm()); } // Create System.Speech recognizer component var recognizer = new SystemSpeechRecognizer( pipeline, new SystemSpeechRecognizerConfiguration() { Language = "en-US", Grammars = new GrammarInfo[] { new GrammarInfo() { Name = Program.AppName, FileName = "SampleGrammar.grxml" }, }, }); // Subscribe the recognizer to the input audio audioInput.PipeTo(recognizer); // Partial and final speech recognition results are posted on the same stream. Here // we use Psi's Where() operator to filter out only the final recognition results. var finalResults = recognizer.Out.Where(result => result.IsFinal); // Print the final recognition result to the console. finalResults.Do(result => { Console.WriteLine($"{result.Text} (confidence: {result.Confidence})"); }); // Create a data store to log the data to if necessary. A data store is necessary // only if output logging is enabled. var dataStore = CreateDataStore(pipeline, outputLogPath); // For disk logging only if (dataStore != null) { // Log the microphone audio and recognition results audioInput.Write($"{Program.AppName}.MicrophoneAudio", dataStore); finalResults.Write($"{Program.AppName}.FinalRecognitionResults", dataStore); } // Register an event handler to catch pipeline errors pipeline.PipelineExceptionNotHandled += Pipeline_PipelineException; // Register an event handler to be notified when the pipeline completes pipeline.PipelineCompleted += Pipeline_PipelineCompleted; // Run the pipeline pipeline.RunAsync(); // The file SampleGrammar.grxml defines a grammar to transcribe numbers Console.WriteLine("Say any number between 0 and 100"); Console.WriteLine("Press any key to exit..."); Console.ReadKey(true); } }
/// <summary> /// Initializes a new instance of the <see cref="RemoteImporter"/> class. /// </summary> /// <param name="pipeline">Pipeline to which to attach.</param> /// <param name="host">Remote host name.</param> /// <param name="port">TCP port on which to connect (default 11411).</param> /// <param name="allowSequenceRestart">Whether to allow sequence ID restarts upon connection loss/reacquire.</param> /// <remarks>In this case the start is a special behavior that is `DateTime.UtcNow` _at the sending `RemoteExporter`_.</remarks> public RemoteImporter(Pipeline pipeline, string host, int port = RemoteExporter.DefaultPort, bool allowSequenceRestart = true) : this(name => PsiStore.Open(pipeline, name, null), new TimeInterval(DateTime.MinValue, DateTime.MaxValue), true, host, port, $"RemoteImporter_{Guid.NewGuid().ToString()}", null, allowSequenceRestart) { }
/// <summary> /// Initializes a new instance of the <see cref="RemoteImporter"/> class. /// </summary> /// <param name="pipeline">Pipeline to which to attach.</param> /// <param name="replay">Time interval to be replayed from remote source.</param> /// <param name="host">Remote host name.</param> /// <param name="port">TCP port on which to connect (default 11411).</param> /// <param name="allowSequenceRestart">Whether to allow sequence ID restarts upon connection loss/reacquire.</param> public RemoteImporter(Pipeline pipeline, TimeInterval replay, string host, int port = RemoteExporter.DefaultPort, bool allowSequenceRestart = true) : this(name => PsiStore.Open(pipeline, name, null), replay, false, host, port, $"RemoteImporter_{Guid.NewGuid().ToString()}", null, allowSequenceRestart) { }
public void CrossFrameworkDeserialize() { if (!File.Exists(OtherTestExe)) { Assert.Inconclusive($"Unable to locate {OtherTestExe} to generate the test store for {nameof(this.CrossFrameworkDeserializeMembers)}."); } this.ExecuteTest(OtherTestExe, $"!{nameof(this.CrossFrameworkSerialize)}"); int intValue = 0; byte byteValue = 0; bool boolValue = false; short shortValue = 0; long longValue = 0; char charValue = '\0'; string stringValue = null; double doubleValue = 0; float floatValue = 0; float[] floatArray = null; List <string> stringList = null; ArraySegment <string> stringArraySegment = default; Queue <TimeSpan> queue = null; EqualityComparer <int> intComparer = null; Tuple <long, string> tuple = null; (DateTime, Stack <int>)valueTuple = default; Array intArray = null; ICollection stringArray = null; IEqualityComparer enumComparer = null; using (var p = Pipeline.Create()) { var store = PsiStore.Open(p, "Store1", this.testPath); store.Serializers.Register <System.Drawing.Point>(); store.OpenStream <int>("int").Do(x => intValue = x); store.OpenStream <byte>("byte").Do(x => byteValue = x); store.OpenStream <bool>("bool").Do(x => boolValue = x); store.OpenStream <short>("short").Do(x => shortValue = x); store.OpenStream <long>("long").Do(x => longValue = x); store.OpenStream <char>("char").Do(x => charValue = x); store.OpenStream <string>("string").Do(x => stringValue = x); store.OpenStream <double>("double").Do(x => doubleValue = x); store.OpenStream <float>("float").Do(x => floatValue = x); store.OpenStream <float[]>("floatArray").Do(x => floatArray = x.DeepClone()); store.OpenStream <List <string> >("stringList").Do(x => stringList = x.DeepClone()); store.OpenStream <ArraySegment <string> >("stringArraySegment").Do(x => stringArraySegment = x.DeepClone()); store.OpenStream <Queue <TimeSpan> >("queue").Do(x => queue = x.DeepClone()); store.OpenStream <EqualityComparer <int> >("intComparer").Do(x => intComparer = x.DeepClone()); store.OpenStream <Tuple <long, string> >("tuple").Do(x => tuple = x.DeepClone()); store.OpenStream <(DateTime, Stack <int>)>("dateStackTuple").Do(x => valueTuple = x.DeepClone()); store.OpenStream <Array>("intArray").Do(x => intArray = x.DeepClone()); store.OpenStream <ICollection>("stringArray").Do(x => stringArray = x.DeepClone()); store.OpenStream <IEqualityComparer>("enumComparer").Do(x => enumComparer = x.DeepClone()); p.Run(); } Assert.AreEqual(0x7777AAA, intValue); Assert.AreEqual(0xBB, byteValue); Assert.AreEqual(true, boolValue); Assert.AreEqual(0x7CDD, shortValue); Assert.AreEqual(0x77777777EEEEEEEE, longValue); Assert.AreEqual('G', charValue); Assert.AreEqual("This is a test.", stringValue); Assert.AreEqual(Math.PI, doubleValue); Assert.AreEqual(-1.234f, floatValue); CollectionAssert.AreEqual(new[] { 0.1f, 2.3f }, floatArray); CollectionAssert.AreEqual(new[] { "one", "two" }, stringList); CollectionAssert.AreEqual(new[] { "bbb", "ccc" }, stringArraySegment.ToArray()); CollectionAssert.AreEqual(new[] { TimeSpan.Zero, TimeSpan.FromSeconds(1) }, queue); Assert.IsTrue(intComparer.Equals(1952, 1952)); Assert.AreEqual(0x77777777EEEEEEEE, tuple.Item1); Assert.AreEqual("This is a tuple.", tuple.Item2); Assert.AreEqual(new DateTime(2020, 1, 2), valueTuple.Item1); CollectionAssert.AreEqual(new[] { 782, 33 }, valueTuple.Item2); CollectionAssert.AreEqual(new[] { 0, 3 }, intArray); CollectionAssert.AreEqual(new[] { "three", "four" }, stringArray); Assert.IsTrue(enumComparer.Equals(DayOfWeek.Friday, DayOfWeek.Friday)); }
/// <summary> /// Initializes a new instance of the <see cref="RemoteExporter"/> class. /// </summary> /// <param name="pipeline">Pipeline to which to attach.</param> /// <param name="port">TCP port on which to listen (default 11411).</param> /// <param name="transport">Transport kind to use.</param> /// <param name="maxBytesPerSecond">Maximum bytes/sec quota (default infinite).</param> /// <param name="bytesPerSecondSmoothingWindowSeconds">Smoothing window over which to compute bytes/sec (default 5 sec.).</param> public RemoteExporter(Pipeline pipeline, int port = DefaultPort, TransportKind transport = DefaultTransport, long maxBytesPerSecond = long.MaxValue, double bytesPerSecondSmoothingWindowSeconds = 5.0) : this(PsiStore.Create(pipeline, $"RemoteExporter_{Guid.NewGuid().ToString()}", null, true), port, transport, maxBytesPerSecond, bytesPerSecondSmoothingWindowSeconds) { }
/// <summary> /// Execute task against each message in a stream. /// </summary> /// <param name="stream">Stream name.</param> /// <param name="store">Store name.</param> /// <param name="path">Store path.</param> /// <param name="name">Task name.</param> /// <param name="assemblies">Optional assemblies containing task.</param> /// <param name="args">Additional configuration arguments.</param> /// <returns>Success flag.</returns> internal static int ExecuteTask(string stream, string store, string path, string name, IEnumerable <string> assemblies, IEnumerable <string> args) { Console.WriteLine($"Execute Task (stream={stream}, store={store}, path={path}, name={name}, assemblies={assemblies}, args={args})"); // find task var tasks = LoadTasks(assemblies).Where(t => t.Attribute.Name == name).ToArray(); if (tasks.Length == 0) { throw new Exception($"Could not find task named '{name}'."); } else if (tasks.Length > 1) { throw new Exception($"Ambiguous task name ({tasks.Count()} tasks found named '{name}')."); } var task = tasks[0]; // process task using (var pipeline = Pipeline.Create()) { var importer = store != null?PsiStore.Open(pipeline, store, Path.GetFullPath(path)) : null; // prepare parameters var streamMode = stream?.Length > 0; var messageIndex = -1; var envelopeIndex = -1; var argList = args.ToArray(); var argIndex = 0; var parameterInfo = task.Method.GetParameters(); var parameters = new object[parameterInfo.Length]; for (var i = 0; i < parameterInfo.Length; i++) { var p = parameterInfo[i]; if (p.ParameterType.IsAssignableFrom(typeof(Importer))) { if (importer == null) { throw new ArgumentException("Error: Task requires a store, but no store argument supplied (-s)."); } parameters[i] = importer; } else if (p.ParameterType.IsAssignableFrom(typeof(Pipeline))) { parameters[i] = pipeline; } else if (p.ParameterType.IsAssignableFrom(typeof(Envelope))) { envelopeIndex = i; } else if (streamMode && messageIndex == -1) { messageIndex = i; // assumed first arg } else { Action <string, Func <string, object> > processArgs = (friendlyName, parser) => { if (argIndex < args.Count()) { // take from command-line args try { parameters[i] = parser(argList[argIndex++]); } catch (Exception ex) { throw new ArgumentException($"Error: Parameter '{p.Name}' ({i}) expected {friendlyName}.", ex); } } else { // get value interactively do { try { Console.Write($"{p.Name} ({friendlyName})? "); parameters[i] = parser(Console.ReadLine()); } catch { Console.WriteLine($"Error: Expected {friendlyName}."); } }while (parameters[i] == null); } }; if (p.ParameterType.IsAssignableFrom(typeof(double))) { processArgs("double", v => double.Parse(v)); } else if (p.ParameterType.IsAssignableFrom(typeof(int))) { processArgs("integer", v => int.Parse(v)); } else if (p.ParameterType.IsAssignableFrom(typeof(bool))) { processArgs("boolean", v => bool.Parse(v)); } else if (p.ParameterType.IsAssignableFrom(typeof(DateTime))) { processArgs("datetime", v => DateTime.Parse(v)); } else if (p.ParameterType.IsAssignableFrom(typeof(TimeSpan))) { processArgs("timespan", v => TimeSpan.Parse(v)); } else if (p.ParameterType.IsAssignableFrom(typeof(string))) { processArgs("string", v => v); } else { throw new ArgumentException($"Unexpected parameter type ({p.ParameterType})."); } } } if (streamMode) { if (importer == null) { throw new ArgumentException("Error: Task requires a stream within a store, but no store argument supplied (-s)."); } importer.OpenDynamicStream(stream).Do((m, e) => { if (messageIndex != -1) { parameters[messageIndex] = m; } if (envelopeIndex != -1) { parameters[envelopeIndex] = e; } task.Method.Invoke(null, parameters); }); } else { task.Method.Invoke(null, parameters); } if (importer != null) { pipeline.ProgressReportInterval = TimeSpan.FromSeconds(1); pipeline.RunAsync(null, new Progress <double>(p => Console.WriteLine($"Progress: {p * 100.0:F2}%"))); pipeline.WaitAll(); } } return(0); }
public async Task TranscribeConversationsAsync(IEnumerable <string> voiceSignatureStringUsers) { uint samplesPerSecond = 16000; byte bitsPerSample = 16; byte channels = 8; // 7 + 1 channels var config = SpeechConfig.FromSubscription(this.SubscriptionKey, this.Region); config.SetProperty("ConversationTranscriptionInRoomAndOnline", "true"); var stopRecognition = new TaskCompletionSource <int>(); using (var audioInput = AudioInputStream.CreatePushStream(AudioStreamFormat.GetWaveFormatPCM(samplesPerSecond, bitsPerSample, channels))) { var meetingID = Guid.NewGuid().ToString(); using (var conversation = await Conversation.CreateConversationAsync(config, meetingID)) { // create a conversation transcriber using audio stream input using (this.conversationTranscriber = new ConversationTranscriber(AudioConfig.FromStreamInput(audioInput))) { conversationTranscriber.Transcribing += (s, e) => { this.SetText($"TRANSCRIBING: Text={e.Result.Text} SpeakerId={e.Result.UserId}"); }; conversationTranscriber.Transcribed += (s, e) => { if (e.Result.Reason == ResultReason.RecognizedSpeech) { this.SetText($"TRANSCRIBED: Text={e.Result.Text} SpeakerId={e.Result.UserId}"); } else if (e.Result.Reason == ResultReason.NoMatch) { this.SetText($"NOMATCH: Speech could not be recognized."); } }; conversationTranscriber.Canceled += (s, e) => { this.SetText($"CANCELED: Reason={e.Reason}"); if (e.Reason == CancellationReason.Error) { this.SetText($"CANCELED: ErrorCode={e.ErrorCode}"); this.SetText($"CANCELED: ErrorDetails={e.ErrorDetails}"); this.SetText($"CANCELED: Did you update the subscription info?"); stopRecognition.TrySetResult(0); } }; conversationTranscriber.SessionStarted += (s, e) => { this.SetText($"\nSession started event. SessionId={e.SessionId}"); }; conversationTranscriber.SessionStopped += (s, e) => { this.SetText($"\nSession stopped event. SessionId={e.SessionId}"); this.SetText("\nStop recognition."); stopRecognition.TrySetResult(0); }; // Add participants to the conversation. int i = 1; foreach (var voiceSignatureStringUser in voiceSignatureStringUsers) { var speaker = Participant.From($"User{i++}", "en-US", voiceSignatureStringUser); await conversation.AddParticipantAsync(speaker); } // Join to the conversation and start transcribing await conversationTranscriber.JoinConversationAsync(conversation); await conversationTranscriber.StartTranscribingAsync().ConfigureAwait(false); using (var p = Pipeline.Create()) { var store = PsiStore.Create(p, "Transcribe", @"D:\Temp"); var capture = new AudioCapture(p, WaveFormat.CreatePcm((int)samplesPerSecond, bitsPerSample, channels)).Write("Audio", store); capture.Do(audio => audioInput.Write(audio.Data)); p.RunAsync(); // waits for completion, then stop transcription await stopRecognition.Task; } await conversationTranscriber.StopTranscribingAsync().ConfigureAwait(false); } } } }
/// <summary> /// This is the main code for our Multimodal Speech Detection demo. /// </summary> private void PerformMultiModalSpeechDetection() { Console.WriteLine("Initializing Psi."); bool detected = false; // First create our \Psi pipeline using (var pipeline = Pipeline.Create("MultiModalSpeechDetection")) { // Register an event handler to catch pipeline errors pipeline.PipelineExceptionNotHandled += Pipeline_PipelineException; // Register an event handler to be notified when the pipeline completes pipeline.PipelineCompleted += Pipeline_PipelineCompleted; // Next create our Kinect sensor. We will be using the color images, face tracking, and audio from the Kinect sensor var kinectSensorConfig = new KinectSensorConfiguration { OutputColor = true, OutputAudio = true, OutputBodies = true, // In order to detect faces using Kinect you must also enable detection of bodies }; var kinectSensor = new KinectSensor(pipeline, kinectSensorConfig); var kinectFaceDetector = new Microsoft.Psi.Kinect.Face.KinectFaceDetector(pipeline, kinectSensor, Microsoft.Psi.Kinect.Face.KinectFaceDetectorConfiguration.Default); // Create our Voice Activation Detector var speechDetector = new SystemVoiceActivityDetector(pipeline); var convertedAudio = kinectSensor.Audio.Resample(WaveFormat.Create16kHz1Channel16BitPcm()); convertedAudio.PipeTo(speechDetector); // Use the Kinect's face track to determine if the mouth is opened var mouthOpenAsFloat = kinectFaceDetector.Faces.Where(faces => faces.Count > 0).Select((List <Microsoft.Psi.Kinect.Face.KinectFace> list) => { if (!detected) { detected = true; Console.WriteLine("Found your face"); } bool open = (list[0] != null) ? list[0].FaceProperties[Microsoft.Kinect.Face.FaceProperty.MouthOpen] == Microsoft.Kinect.DetectionResult.Yes : false; return(open ? 1.0 : 0.0); }); // Next take the "mouthOpen" value and create a hold on that value (so that we don't see 1,0,1,0,1 but instead would see 1,1,1,1,0.8,0.6,0.4) var mouthOpen = mouthOpenAsFloat.Hold(0.1); // Next join the results of the speechDetector with the mouthOpen generator and only select samples where // we have detected speech and that the mouth was open. var mouthAndSpeechDetector = speechDetector.Join(mouthOpen, hundredMs).Select((t, e) => t.Item1 && t.Item2); // Convert our speech into text var speechRecognition = convertedAudio.SpeechToText(mouthAndSpeechDetector); speechRecognition.Do((s, t) => { if (s.Item1.Length > 0) { Console.WriteLine("You said: " + s.Item1); } }); // Create a stream of landmarks (points) from the face detector var facePoints = new List <Tuple <System.Windows.Point, string> >(); var landmarks = kinectFaceDetector.Faces.Where(faces => faces.Count > 0).Select((List <Microsoft.Psi.Kinect.Face.KinectFace> list) => { facePoints.Clear(); System.Windows.Point pt1 = new System.Windows.Point( list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.EyeLeft].X, list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.EyeLeft].Y); facePoints.Add(Tuple.Create(pt1, string.Empty)); System.Windows.Point pt2 = new System.Windows.Point( list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.EyeRight].X, list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.EyeRight].Y); facePoints.Add(Tuple.Create(pt2, string.Empty)); System.Windows.Point pt3 = new System.Windows.Point( list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.MouthCornerLeft].X, list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.MouthCornerLeft].Y); facePoints.Add(Tuple.Create(pt3, string.Empty)); System.Windows.Point pt4 = new System.Windows.Point( list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.MouthCornerRight].X, list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.MouthCornerRight].Y); facePoints.Add(Tuple.Create(pt4, string.Empty)); System.Windows.Point pt5 = new System.Windows.Point( list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.Nose].X, list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.Nose].Y); facePoints.Add(Tuple.Create(pt5, string.Empty)); return(facePoints); }); // ******************************************************************** // Finally create a Live Visualizer using PsiStudio. // We must persist our streams to a store in order for Live Viz to work properly // ******************************************************************** // Create store for the data. Live Visualizer can only read data from a store. var pathToStore = Environment.GetFolderPath(Environment.SpecialFolder.MyVideos); var store = PsiStore.Create(pipeline, ApplicationName, pathToStore); mouthOpen.Select(v => v ? 1d : 0d).Write("MouthOpen", store); speechDetector.Select(v => v ? 1d : 0d).Write("VAD", store); mouthAndSpeechDetector.Write("Join(MouthOpen,VAD)", store); kinectSensor.Audio.Write("Audio", store); var images = kinectSensor.ColorImage.EncodeJpeg(90, DeliveryPolicy.LatestMessage).Out; images.Write("Images", store, true, DeliveryPolicy.LatestMessage); landmarks.Write("FaceLandmarks", store); // Run the pipeline pipeline.RunAsync(); Console.WriteLine("Press any key to finish recording"); Console.ReadKey(); } }
public void CreateAndStartPipeline() { this.pipeline = Pipeline.Create("Teams Pipeline", enableDiagnostics: this.botSettings.EnablePsiDiagnostics); this.frameSourceComponent = new FrameSourceComponent(this.pipeline, logger); var mpegConfig = Mpeg4WriterConfiguration.Default; mpegConfig.ContainsAudio = false; mpegConfig.ImageWidth = (uint)this.botSettings.Resize.Width; mpegConfig.ImageHeight = (uint)this.botSettings.Resize.Height; mpegConfig.PixelFormat = PixelFormat.BGR_24bpp; var basePath = this.botSettings.RecordingFilePath; var endpointUrl = this.botSettings.ModelEndpointUrl; var resized = frameSourceComponent .Video .Select(v => v.First().Value) // discarding participant id - this means that no information of participant is carried forward .Resize(this.botSettings.Resize.Width, this.botSettings.Resize.Height) .Name("Resized Frames"); // input is stream of frames var fileNames = resized .WriteMP4InBatches(TimeSpan.FromSeconds(this.botSettings.VideoSegmentationIntervalInSeconds), basePath, mpegConfig) .Name("FileNames"); var labelStream = fileNames .CallModel(endpointUrl, basePath, logger).Name("Model Result") .Do(l => this.logger.Info($"file: {l.filename} label: {l.label}")); // output is stream of labels labelStream.Item2() .PerformTextToSpeech(this.ttsSettings, this.logger).Name("Text To Speech") .Do(bytes => this.sendAudioToBot(CreateAudioMediaBuffers(DateTime.UtcNow.Ticks, bytes))).Name("Send Audio To Bot"); Generators .Repeat(pipeline, true, TimeSpan.FromSeconds(1.0 / 15)).Name("15fps generation event") .Pair(labelStream, DeliveryPolicy.LatestMessage, DeliveryPolicy.LatestMessage) .Do(f => { try { var text = f.Item3; using (var sharedImage = ProduceScreenShare(text)) { var image = sharedImage.Resource; var nv12 = BGRAtoNV12(image.ImageData, image.Width, image.Height); this.sendScreenShareToBot(nv12); } } catch (Exception ex) { this.logger.Error(ex, "Error while screen sharing"); } }).Name("Screen Share to bot"); if (this.botSettings.EnablePsiStore) { var store = PsiStore.Create(pipeline, "Bot", this.botSettings.PsiStorePath); //resized.Write("video", store); labelStream.Write("label", store); if (this.botSettings.EnablePsiDiagnostics) { pipeline.Diagnostics.Write("Diagnostics", store); } } this.pipeline.PipelineExceptionNotHandled += (_, ex) => { this.logger.Error(ex.Exception, $"PSI PIPELINE ERROR: {ex.Exception.Message}"); }; pipeline.RunAsync(); //Task.Run(async () => { // await Task.Delay(30000); // this.logger.Warn("STOPPPPPPPPPIIIIIIIIINNNNNNNNGGGGGGGGGGGGGGGGG"); // pipeline.Dispose(); //}); }